Skip to content

Commit bd46c46

Browse files
houjunnickaruwanggithub-actions[bot]jeanbez
authored
Fix pdc ls (#154)
* pdc import, export, ls compiled successfully * removed requested files * formatting issues * changed install tools * gets checkpoint files * grabbing checkpoint files from within sub-directories, minor comments * Committing clang-format changes * Committing clang-format changes * Fix a few issues with pdc_ls * Committing clang-format changes --------- Co-authored-by: nickaruwang <[email protected]> Co-authored-by: Nick Wang <[email protected]> Co-authored-by: github-actions <github-actions[bot]@users.noreply.github.com> Co-authored-by: Jean Luca Bez <[email protected]>
1 parent 1b6d60d commit bd46c46

File tree

1 file changed

+97
-28
lines changed

1 file changed

+97
-28
lines changed

‎src/tools/pdc_ls.c

Lines changed: 97 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -114,27 +114,89 @@ int pdc_server_rank_g = 0;
114114
int pdc_server_size_g = 1;
115115
double total_mem_usage_g = 0.0;
116116

117+
int
118+
isDir(const char *fileName)
119+
{
120+
struct stat path;
121+
stat(fileName, &path);
122+
return S_ISREG(path.st_mode);
123+
}
124+
117125
static void pdc_ls(FileNameNode *file_name_node, int argc, char *argv[]);
118126

119127
int
120128
main(int argc, char *argv[])
121129
{
122130
if (argc == 1) {
123-
printf("Expected directory/checkpoint file.\n");
124-
return 1;
131+
printf("Usage: ./pdc_ls pdc_checkpoint_directory/file [-n obj_name] [-i obj_id] [-json json_fname] "
132+
"[-ln (list all names)] [-ls (list all ids)] [-s (summary)]\n");
133+
return 0;
125134
}
126135
else {
127136
FileNameNode * head = NULL;
128137
FileNameNode * cur_node = NULL;
129138
DIR * d;
130139
struct dirent *dir;
140+
struct dirent *direc;
131141
d = opendir(argv[1]);
142+
char *full_path;
132143

133144
if (d) {
134145
while ((dir = readdir(d)) != NULL) {
146+
// if it's directory
147+
if (!isDir(dir->d_name)) {
148+
if (strstr(dir->d_name, ".")) {
149+
// ignore parent and current directories
150+
continue;
151+
}
152+
// appends path together
153+
char tmp[1024];
154+
sprintf(tmp, "%s/%s", argv[1], dir->d_name);
155+
DIR *d1 = opendir(tmp);
156+
/* printf("%s\n", tmp); */
157+
158+
while ((direc = readdir(d1)) != NULL) { // go into it and go for checkpoint files again
159+
if (strstr(direc->d_name, "metadata_checkpoint.")) {
160+
// printf("getting checkpoints\n");
161+
char last = argv[1][strlen(argv[1]) - 1];
162+
if (last == '/') {
163+
full_path = (char *)malloc(sizeof(char) *
164+
(strlen(argv[1]) + strlen(direc->d_name) + 1));
165+
strcpy(full_path, argv[1]);
166+
strcat(full_path, direc->d_name);
167+
strcat(full_path, "/");
168+
strcat(full_path, direc->d_name);
169+
}
170+
else {
171+
full_path = (char *)malloc(sizeof(char) *
172+
(strlen(argv[1]) + strlen(direc->d_name) + 2));
173+
strcpy(full_path, argv[1]);
174+
strcat(full_path, "/");
175+
strcat(full_path, dir->d_name);
176+
strcat(full_path, "/");
177+
strcat(full_path, direc->d_name);
178+
}
179+
if (head == NULL) {
180+
FileNameNode *new_node = (FileNameNode *)malloc(sizeof(FileNameNode));
181+
new_node->file_name = full_path;
182+
new_node->next = NULL;
183+
head = new_node;
184+
cur_node = new_node;
185+
}
186+
else {
187+
FileNameNode *new_node = (FileNameNode *)malloc(sizeof(FileNameNode));
188+
new_node->file_name = full_path;
189+
new_node->next = NULL;
190+
cur_node->next = new_node;
191+
cur_node = new_node;
192+
}
193+
}
194+
}
195+
closedir(d1);
196+
}
135197
if (strstr(dir->d_name, "metadata_checkpoint.")) {
136-
char last = argv[1][strlen(argv[1]) - 1];
137-
char *full_path;
198+
printf("%s\n", dir->d_name);
199+
char last = argv[1][strlen(argv[1]) - 1];
138200
if (last == '/') {
139201
full_path =
140202
(char *)malloc(sizeof(char) * (strlen(argv[1]) + strlen(dir->d_name) + 1));
@@ -163,14 +225,15 @@ main(int argc, char *argv[])
163225
cur_node = new_node;
164226
}
165227
}
166-
}
228+
} // End if dir = readdir
167229
closedir(d);
168-
}
230+
} // Ene if d
169231
else {
232+
// Open one checkpoint file
170233
FILE *file = fopen(argv[1], "r");
171234
if (file != NULL) {
172-
FileNameNode *new_node = (FileNameNode *)malloc(sizeof(FileNameNode));
173-
char * full_path = (char *)malloc(sizeof(char) * (strlen(argv[1]) + 1));
235+
FileNameNode *new_node = (FileNameNode *)malloc(sizeof(FileNameNode));
236+
full_path = (char *)malloc(sizeof(char) * (strlen(argv[1]) + 1));
174237
strcpy(full_path, argv[1]);
175238
new_node->file_name = full_path;
176239
new_node->next = NULL;
@@ -179,9 +242,10 @@ main(int argc, char *argv[])
179242
fclose(file);
180243
}
181244
}
245+
182246
if (head == NULL) {
183247
printf("Unable to open/locate checkpoint file(s).\n");
184-
return 1;
248+
return -1;
185249
}
186250
else {
187251
pdc_ls(head, argc, argv);
@@ -311,6 +375,7 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
311375
int list_names = 0;
312376
int list_ids = 0;
313377
int summary = 0;
378+
int print_all = 1;
314379

315380
int arg_index = 2;
316381
while (arg_index < argc) {
@@ -332,12 +397,15 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
332397
}
333398
else if (strcmp(argv[arg_index], "-ln") == 0) {
334399
list_names = 1;
400+
print_all = 0;
335401
}
336402
else if (strcmp(argv[arg_index], "-li") == 0) {
337-
list_ids = 1;
403+
list_ids = 1;
404+
print_all = 0;
338405
}
339406
else if (strcmp(argv[arg_index], "-s") == 0) {
340-
summary = 1;
407+
summary = 1;
408+
print_all = 0;
341409
}
342410
arg_index++;
343411
}
@@ -365,7 +433,7 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
365433
while (cur_file_node != NULL) {
366434
filename = cur_file_node->file_name;
367435
stat(filename, &attr);
368-
printf("[INFO] File [%s] last modified at: %s\n", filename, ctime(&attr.st_mtime));
436+
printf("[INFO] File [%s] last modified at: %s", filename, ctime(&attr.st_mtime));
369437
// Start server restart code
370438
perr_t ret_value = SUCCEED;
371439
int n_entry, count, i, j, nobj = 0, all_nobj = 0, all_n_region, n_region, n_objs, total_region = 0,
@@ -387,19 +455,21 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
387455
}
388456

389457
if (fread(&n_cont, sizeof(int), 1, file) != 1) {
390-
printf("Read failed for n_count\n");
458+
printf("Read failed for cont count\n");
391459
}
392460
all_cont = n_cont;
393461
while (n_cont > 0) {
394462
hash_key = (uint32_t *)malloc(sizeof(uint32_t));
395463
if (fread(hash_key, sizeof(uint32_t), 1, file) != 1) {
396-
printf("Read failed for hash_key\n");
464+
printf("Read failed for cont hash_key\n");
465+
return;
397466
}
398467

399468
// Reconstruct hash table
400469
cont_entry = (pdc_cont_hash_table_entry_t *)malloc(sizeof(pdc_cont_hash_table_entry_t));
401470
if (fread(cont_entry, sizeof(pdc_cont_hash_table_entry_t), 1, file) != 1) {
402471
printf("Read failed for cont_entry\n");
472+
return;
403473
}
404474

405475
n_cont--;
@@ -410,12 +480,14 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
410480

411481
while (n_entry > 0) {
412482
if (fread(&count, sizeof(int), 1, file) != 1) {
413-
printf("Read failed for count\n");
483+
printf("Read failed for obj count\n");
484+
return;
414485
}
415486

416487
hash_key = (uint32_t *)malloc(sizeof(uint32_t));
417488
if (fread(hash_key, sizeof(uint32_t), 1, file) != 1) {
418-
printf("Read failed for hash_key\n");
489+
printf("Read failed for obj hash_key\n");
490+
return;
419491
}
420492

421493
// Reconstruct hash table
@@ -481,6 +553,9 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
481553
if (fread(&kvtag_list->kvtag->size, sizeof(uint32_t), 1, file) != 1) {
482554
printf("Read failed for kvtag_list->kvtag->size\n");
483555
}
556+
if (fread(&kvtag_list->kvtag->type, sizeof(int8_t), 1, file) != 1) {
557+
printf("Read failed for kvtag_list->kvtag->size\n");
558+
}
484559
kvtag_list->kvtag->value = malloc(kvtag_list->kvtag->size);
485560
if (fread(kvtag_list->kvtag->value, kvtag_list->kvtag->size, 1, file) != 1) {
486561
printf("Read failed for kvtag_list->kvtag->value\n");
@@ -653,11 +728,11 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
653728
cJSON * start_arr_json = NULL;
654729
cJSON * output = cJSON_CreateObject();
655730
int prev_cont_id = -1;
731+
char buf[1024];
656732
while (cur_m_node != NULL) {
657733
cur_metadata = cur_m_node->metadata_ptr;
658734
if (prev_cont_id != cur_metadata->cont_id) {
659735
cont_id_json = cJSON_CreateArray();
660-
char buf[20];
661736
sprintf(buf, "cont_id: %d", cur_metadata->cont_id);
662737
cJSON_AddItemToObject(output, buf, cont_id_json);
663738
}
@@ -678,7 +753,6 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
678753
}
679754
}
680755

681-
char buf[12];
682756
sprintf(buf, "%d", wanted_id);
683757
reti = regcomp(&regex, buf, 0);
684758
if (reti) {
@@ -687,7 +761,6 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
687761
}
688762
}
689763
else {
690-
// char buf[12];
691764
sprintf(buf, "%d", cur_metadata->obj_id);
692765
reti = regexec(&regex, buf, 0, NULL, 0);
693766
if (!reti) {
@@ -715,8 +788,7 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
715788
add_obj = matched_name;
716789
}
717790
else if (wanted_id) {
718-
int matched_id = 0;
719-
char buf[12];
791+
int matched_id = 0;
720792
sprintf(buf, "%d", wanted_id);
721793
reti = regcomp(&regex, buf, 0);
722794
if (reti) {
@@ -725,7 +797,6 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
725797
}
726798
}
727799
else {
728-
// char buf[12];
729800
sprintf(buf, "%d", cur_metadata->obj_id);
730801
reti = regexec(&regex, buf, 0, NULL, 0);
731802
if (!reti) {
@@ -739,7 +810,6 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
739810
add(obj_names, cur_metadata->obj_name);
740811
}
741812
if (list_ids) {
742-
char buf[12];
743813
sprintf(buf, "%d", cur_metadata->obj_id);
744814
add(obj_ids, buf);
745815
}
@@ -768,8 +838,6 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
768838
cJSON_AddStringToObject(region_info_json, "storage_loc", cur_region->storage_location);
769839
cJSON_AddNumberToObject(region_info_json, "offset", cur_region->offset);
770840
cJSON_AddNumberToObject(region_info_json, "num_dims", cur_region->ndim);
771-
// FIXME: statement with no effect. what did we expect to do here?
772-
// dims[cur_region->ndim];
773841
for (int i = 0; i < (cur_metadata->ndim); i++) {
774842
dims[i] = (cur_region->start)[i];
775843
}
@@ -798,13 +866,13 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
798866
FILE *fp;
799867
if (output_file_name) {
800868
fp = fopen(output_file_name, "w");
869+
printf("Output to [%s]\n", output_file_name);
801870
}
802871
else {
803872
fp = stdout;
804873
}
805874
if (list_names) {
806-
cJSON *all_names_json =
807-
cJSON_CreateStringArray((const char *const *)obj_names->items, obj_names->length);
875+
cJSON *all_names_json = cJSON_CreateStringArray((const char **)obj_names->items, obj_names->length);
808876
cJSON_AddItemToObject(output, "all_obj_names", all_names_json);
809877
}
810878
if (list_ids) {
@@ -816,7 +884,6 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
816884
cJSON_AddItemToObject(output, "all_obj_ids", all_ids_json);
817885
}
818886
if (summary) {
819-
char buf[100];
820887
sprintf(buf, "pdc_ls found: %d containers, %d objects, %d regions", all_cont_total, all_nobj_total,
821888
all_n_region_total);
822889
cJSON_AddStringToObject(output, "summary", buf);
@@ -827,4 +894,6 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[])
827894
char *json_string = cJSON_Print(output);
828895
fprintf(fp, json_string);
829896
fprintf(fp, "\n");
897+
if (output_file_name)
898+
fclose(fp);
830899
}

0 commit comments

Comments
 (0)