From fc3a57015f0f06e94ae859ef2e233746c37c204c Mon Sep 17 00:00:00 2001 From: yucongalicechen Date: Tue, 14 May 2024 21:17:16 -0400 Subject: [PATCH 1/7] initial commit on implementing a wildcard pattern for input --- src/diffpy/labpdfproc/labpdfprocapp.py | 9 ++++++--- src/diffpy/labpdfproc/tests/test_tools.py | 19 +++++++++++++++++++ src/diffpy/labpdfproc/tools.py | 11 ++++++++++- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/diffpy/labpdfproc/labpdfprocapp.py b/src/diffpy/labpdfproc/labpdfprocapp.py index 6dca178..bff4143 100644 --- a/src/diffpy/labpdfproc/labpdfprocapp.py +++ b/src/diffpy/labpdfproc/labpdfprocapp.py @@ -29,7 +29,10 @@ def get_args(override_cli_inputs=None): "'.' (load everything in the current directory), 'data' (load" "everything in the folder ./data), 'data/file_list.txt' (load" " the list of files contained in the text-file called " - "file_list.txt that can be found in the folder ./data).", + "file_list.txt that can be found in the folder ./data). " + "Wildcard character (*) is accepted. Examples include './*chi'" + " (load all files with .chi extension) and 'data/test*' (load " + "all files starting with 'test' in the folder ./data). ", ) p.add_argument( "-a", @@ -101,7 +104,7 @@ def main(): args.wavelength = set_wavelength(args) args = load_user_metadata(args) - for filepath in args.input_directory: + for filepath in args.input_paths: outfilestem = filepath.stem + "_corrected" corrfilestem = filepath.stem + "_cve" outfile = args.output_directory / (outfilestem + ".chi") @@ -125,7 +128,7 @@ def main(): yarray, "tth", scat_quantity="x-ray", - name=str(args.input_file), + name=filepath.stem, metadata={"muD": args.mud, "anode_type": args.anode_type}, ) diff --git a/src/diffpy/labpdfproc/tests/test_tools.py b/src/diffpy/labpdfproc/tests/test_tools.py index 23e56bb..29618ac 100644 --- a/src/diffpy/labpdfproc/tests/test_tools.py +++ b/src/diffpy/labpdfproc/tests/test_tools.py @@ -54,6 +54,25 @@ ["input_dir/file_list_example2.txt"], ["input_dir/good_data.chi", "good_data.xy", "input_dir/good_data.txt"], ), + ( # wildcard pattern, same directory + ["./*.chi"], + ["good_data.chi"], + ), + ( # wildcard pattern, input directory + ["input_dir/*.chi"], + ["input_dir/good_data.chi"], + ), + ( # mixture of valid wildcard patterns + ["good_data*", "./*.pkl", "unreadable*.txt", "input_dir/*.chi"], + [ + "good_data.chi", + "good_data.xy", + "good_data.txt", + "unreadable_file.txt", + "binary.pkl", + "input_dir/good_data.chi", + ], + ), ] diff --git a/src/diffpy/labpdfproc/tools.py b/src/diffpy/labpdfproc/tools.py index a95bf92..fda8456 100644 --- a/src/diffpy/labpdfproc/tools.py +++ b/src/diffpy/labpdfproc/tools.py @@ -86,7 +86,16 @@ def set_input_lists(args): f"Cannot find {input_name}. Please specify valid input file(s) or directories." ) else: - raise FileNotFoundError(f"Cannot find {input_name}") + if "*" in input_name.split("/")[-1] and input_name.count("*") == 1: + input_parent_directory = input_path.parents[0] + input_pattern = input_path.relative_to(input_parent_directory) + input_files = Path(input_parent_directory).glob(str(input_pattern)) + input_files = [ + file.resolve() for file in input_files if file.is_file() and "file_list" not in file.name + ] + input_paths.extend(input_files) + else: + raise FileNotFoundError(f"Cannot find {input_name}") setattr(args, "input_paths", list(set(input_paths))) return args From a02a085b3ccbcb04b10f3c9fd808ccd1dee58c9d Mon Sep 17 00:00:00 2001 From: yucongalicechen Date: Tue, 14 May 2024 21:34:16 -0400 Subject: [PATCH 2/7] relaxed wildcard check condition --- src/diffpy/labpdfproc/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffpy/labpdfproc/tools.py b/src/diffpy/labpdfproc/tools.py index fda8456..aeaf65c 100644 --- a/src/diffpy/labpdfproc/tools.py +++ b/src/diffpy/labpdfproc/tools.py @@ -86,7 +86,7 @@ def set_input_lists(args): f"Cannot find {input_name}. Please specify valid input file(s) or directories." ) else: - if "*" in input_name.split("/")[-1] and input_name.count("*") == 1: + if "*" in input_name: input_parent_directory = input_path.parents[0] input_pattern = input_path.relative_to(input_parent_directory) input_files = Path(input_parent_directory).glob(str(input_pattern)) From f0ffd0f7ee0b1b8919edad277ab0c771bc9aa71d Mon Sep 17 00:00:00 2001 From: yucongalicechen Date: Wed, 15 May 2024 12:54:51 -0400 Subject: [PATCH 3/7] added pre-filter for wildcard patterns and tests, included more examples for help message --- src/diffpy/labpdfproc/labpdfprocapp.py | 10 +++-- src/diffpy/labpdfproc/tests/test_tools.py | 44 ++++++++++++++++----- src/diffpy/labpdfproc/tools.py | 47 ++++++++++++++++++----- 3 files changed, 78 insertions(+), 23 deletions(-) diff --git a/src/diffpy/labpdfproc/labpdfprocapp.py b/src/diffpy/labpdfproc/labpdfprocapp.py index bff4143..505aa01 100644 --- a/src/diffpy/labpdfproc/labpdfprocapp.py +++ b/src/diffpy/labpdfproc/labpdfprocapp.py @@ -30,9 +30,13 @@ def get_args(override_cli_inputs=None): "everything in the folder ./data), 'data/file_list.txt' (load" " the list of files contained in the text-file called " "file_list.txt that can be found in the folder ./data). " - "Wildcard character (*) is accepted. Examples include './*chi'" - " (load all files with .chi extension) and 'data/test*' (load " - "all files starting with 'test' in the folder ./data). ", + "\nWildcard character (*) is accepted. Examples include './*.chi'" + " (load all files with .chi extension), 'data/*.chi' (load all " + "files in 'data' file with .chi extension), 'file*.chi' (load all " + "files starting with 'file' and ending with .chi extension), 'test*' " + "(load all files and directories starting with 'test'), 'test*/*.chi' " + "(load all directories starting with 'test' and all files under " + "with .chi extension). ", ) p.add_argument( "-a", diff --git a/src/diffpy/labpdfproc/tests/test_tools.py b/src/diffpy/labpdfproc/tests/test_tools.py index 29618ac..63c6b6c 100644 --- a/src/diffpy/labpdfproc/tests/test_tools.py +++ b/src/diffpy/labpdfproc/tests/test_tools.py @@ -7,6 +7,7 @@ from diffpy.labpdfproc.labpdfprocapp import get_args from diffpy.labpdfproc.tools import ( expand_list_file, + expand_wildcard_file, known_sources, load_user_metadata, set_input_lists, @@ -54,25 +55,36 @@ ["input_dir/file_list_example2.txt"], ["input_dir/good_data.chi", "good_data.xy", "input_dir/good_data.txt"], ), - ( # wildcard pattern, same directory + ( # wildcard pattern, matching files with .chi extension in the same directory ["./*.chi"], ["good_data.chi"], ), - ( # wildcard pattern, input directory + ( # wildcard pattern, matching files with .chi extension in the input directory ["input_dir/*.chi"], ["input_dir/good_data.chi"], ), - ( # mixture of valid wildcard patterns - ["good_data*", "./*.pkl", "unreadable*.txt", "input_dir/*.chi"], + ( # wildcard pattern, matching files starting with good_data + ["good_data*"], + ["good_data.chi", "good_data.xy", "good_data.txt"], + ), + ( # wildcard pattern, matching files or directories starting with input + ["input*"], [ - "good_data.chi", - "good_data.xy", - "good_data.txt", - "unreadable_file.txt", - "binary.pkl", "input_dir/good_data.chi", + "input_dir/good_data.xy", + "input_dir/good_data.txt", + "input_dir/unreadable_file.txt", + "input_dir/binary.pkl", ], ), + ( # wildcard pattern, matching files or directories starting with unreadable and ending with .txt extension + ["unreadable*.txt"], + ["unreadable_file.txt"], + ), + ( # wildcard pattern, matching directories starting with input and all files under with .chi extension + ["input*/*.chi"], + ["input_dir/good_data.chi"], + ), ] @@ -84,6 +96,7 @@ def test_set_input_lists(inputs, expected, user_filesystem): cli_inputs = ["2.5"] + inputs actual_args = get_args(cli_inputs) + actual_args = expand_wildcard_file(actual_args) actual_args = expand_list_file(actual_args) actual_args = set_input_lists(actual_args) assert sorted(actual_args.input_paths) == sorted(expected_paths) @@ -108,6 +121,16 @@ def test_set_input_lists(inputs, expected, user_filesystem): ["input_dir/file_list.txt"], "Cannot find missing_file.txt. Please specify valid input file(s) or directories.", ), + ( # valid wildcard pattern, but does not match any files or directories + ["non_existing_dir*"], + "Invalid wildcard input non_existing_dir*. " + "Please ensure the wildcard pattern matches at least one file or directory.", + ), + ( # invalid wildcard pattern + ["invalid_dir**"], + "Invalid wildcard input invalid_dir**. " + "Please ensure the wildcard pattern matches at least one file or directory.", + ), ] @@ -117,8 +140,9 @@ def test_set_input_files_bad(inputs, msg, user_filesystem): os.chdir(base_dir) cli_inputs = ["2.5"] + inputs actual_args = get_args(cli_inputs) - actual_args = expand_list_file(actual_args) with pytest.raises(FileNotFoundError, match=msg[0]): + actual_args = expand_wildcard_file(actual_args) + actual_args = expand_list_file(actual_args) actual_args = set_input_lists(actual_args) diff --git a/src/diffpy/labpdfproc/tools.py b/src/diffpy/labpdfproc/tools.py index aeaf65c..2063090 100644 --- a/src/diffpy/labpdfproc/tools.py +++ b/src/diffpy/labpdfproc/tools.py @@ -1,3 +1,4 @@ +import glob from pathlib import Path WAVELENGTHS = {"Mo": 0.71, "Ag": 0.59, "Cu": 1.54} @@ -28,6 +29,41 @@ def set_output_directory(args): return output_dir +def expand_wildcard_file(args): + """ + Expands wildcard inputs by adding all files or directories within directories matching the pattern. + + Parameters + ---------- + args argparse.Namespace + the arguments from the parser + + Returns + ------- + the arguments with the wildcard inputs expanded + + """ + wildcard_inputs = [input_name for input_name in args.input if "*" in input_name] + for wildcard_input in wildcard_inputs: + if not glob.glob(wildcard_input): + raise FileNotFoundError( + f"Invalid wildcard input {wildcard_input}. " + f"Please ensure the wildcard pattern matches at least one file or directory." + ) + input_files = Path(".").glob(wildcard_input) + for input_file in input_files: + if input_file.is_file(): + args.input.append(str(input_file)) + elif input_file.is_dir(): + files = input_file.glob("*") + inputs = [str(file) for file in files if file.is_file() and "file_list" not in file.name] + args.input.extend(inputs) + else: + raise FileNotFoundError(f"Invalid wildcard input {wildcard_input}.") + args.input.remove(wildcard_input) + return args + + def expand_list_file(args): """ Expands the list of inputs by adding files from file lists and removing the file list. @@ -86,16 +122,7 @@ def set_input_lists(args): f"Cannot find {input_name}. Please specify valid input file(s) or directories." ) else: - if "*" in input_name: - input_parent_directory = input_path.parents[0] - input_pattern = input_path.relative_to(input_parent_directory) - input_files = Path(input_parent_directory).glob(str(input_pattern)) - input_files = [ - file.resolve() for file in input_files if file.is_file() and "file_list" not in file.name - ] - input_paths.extend(input_files) - else: - raise FileNotFoundError(f"Cannot find {input_name}") + raise FileNotFoundError(f"Cannot find {input_name}.") setattr(args, "input_paths", list(set(input_paths))) return args From 0c19e3d0e12959946eb91681b692733f30418d97 Mon Sep 17 00:00:00 2001 From: yucongalicechen Date: Wed, 15 May 2024 15:39:19 -0400 Subject: [PATCH 4/7] added pre-filters to expand_user_input private function, removed unnecessary test cases --- src/diffpy/labpdfproc/labpdfprocapp.py | 15 +++----- src/diffpy/labpdfproc/tests/test_tools.py | 20 ----------- src/diffpy/labpdfproc/tools.py | 44 +++++------------------ 3 files changed, 12 insertions(+), 67 deletions(-) diff --git a/src/diffpy/labpdfproc/labpdfprocapp.py b/src/diffpy/labpdfproc/labpdfprocapp.py index 505aa01..88871c6 100644 --- a/src/diffpy/labpdfproc/labpdfprocapp.py +++ b/src/diffpy/labpdfproc/labpdfprocapp.py @@ -3,7 +3,6 @@ from diffpy.labpdfproc.functions import apply_corr, compute_cve from diffpy.labpdfproc.tools import ( - expand_list_file, known_sources, load_user_metadata, set_input_lists, @@ -29,14 +28,9 @@ def get_args(override_cli_inputs=None): "'.' (load everything in the current directory), 'data' (load" "everything in the folder ./data), 'data/file_list.txt' (load" " the list of files contained in the text-file called " - "file_list.txt that can be found in the folder ./data). " - "\nWildcard character (*) is accepted. Examples include './*.chi'" - " (load all files with .chi extension), 'data/*.chi' (load all " - "files in 'data' file with .chi extension), 'file*.chi' (load all " - "files starting with 'file' and ending with .chi extension), 'test*' " - "(load all files and directories starting with 'test'), 'test*/*.chi' " - "(load all directories starting with 'test' and all files under " - "with .chi extension). ", + "file_list.txt that can be found in the folder ./data), " + "'data/*.chi' (load all files with extension .chi in the " + "folder ./data), 'data*' (load all files and directories starting with 'data').", ) p.add_argument( "-a", @@ -102,7 +96,6 @@ def get_args(override_cli_inputs=None): def main(): args = get_args() - args = expand_list_file(args) args = set_input_lists(args) args.output_directory = set_output_directory(args) args.wavelength = set_wavelength(args) @@ -126,7 +119,7 @@ def main(): ) input_pattern = Diffraction_object(wavelength=args.wavelength) - xarray, yarray = loadData(args.input_file, unpack=True) + xarray, yarray = loadData(filepath, unpack=True) input_pattern.insert_scattering_quantity( xarray, yarray, diff --git a/src/diffpy/labpdfproc/tests/test_tools.py b/src/diffpy/labpdfproc/tests/test_tools.py index 63c6b6c..926808b 100644 --- a/src/diffpy/labpdfproc/tests/test_tools.py +++ b/src/diffpy/labpdfproc/tests/test_tools.py @@ -6,8 +6,6 @@ from diffpy.labpdfproc.labpdfprocapp import get_args from diffpy.labpdfproc.tools import ( - expand_list_file, - expand_wildcard_file, known_sources, load_user_metadata, set_input_lists, @@ -81,10 +79,6 @@ ["unreadable*.txt"], ["unreadable_file.txt"], ), - ( # wildcard pattern, matching directories starting with input and all files under with .chi extension - ["input*/*.chi"], - ["input_dir/good_data.chi"], - ), ] @@ -96,8 +90,6 @@ def test_set_input_lists(inputs, expected, user_filesystem): cli_inputs = ["2.5"] + inputs actual_args = get_args(cli_inputs) - actual_args = expand_wildcard_file(actual_args) - actual_args = expand_list_file(actual_args) actual_args = set_input_lists(actual_args) assert sorted(actual_args.input_paths) == sorted(expected_paths) @@ -121,16 +113,6 @@ def test_set_input_lists(inputs, expected, user_filesystem): ["input_dir/file_list.txt"], "Cannot find missing_file.txt. Please specify valid input file(s) or directories.", ), - ( # valid wildcard pattern, but does not match any files or directories - ["non_existing_dir*"], - "Invalid wildcard input non_existing_dir*. " - "Please ensure the wildcard pattern matches at least one file or directory.", - ), - ( # invalid wildcard pattern - ["invalid_dir**"], - "Invalid wildcard input invalid_dir**. " - "Please ensure the wildcard pattern matches at least one file or directory.", - ), ] @@ -141,8 +123,6 @@ def test_set_input_files_bad(inputs, msg, user_filesystem): cli_inputs = ["2.5"] + inputs actual_args = get_args(cli_inputs) with pytest.raises(FileNotFoundError, match=msg[0]): - actual_args = expand_wildcard_file(actual_args) - actual_args = expand_list_file(actual_args) actual_args = set_input_lists(actual_args) diff --git a/src/diffpy/labpdfproc/tools.py b/src/diffpy/labpdfproc/tools.py index 2063090..cc59c07 100644 --- a/src/diffpy/labpdfproc/tools.py +++ b/src/diffpy/labpdfproc/tools.py @@ -1,4 +1,3 @@ -import glob from pathlib import Path WAVELENGTHS = {"Mo": 0.71, "Ag": 0.59, "Cu": 1.54} @@ -29,9 +28,9 @@ def set_output_directory(args): return output_dir -def expand_wildcard_file(args): +def _expand_user_input(args): """ - Expands wildcard inputs by adding all files or directories within directories matching the pattern. + Expands the list of inputs by adding files from file lists and wildcards. Parameters ---------- @@ -40,45 +39,17 @@ def expand_wildcard_file(args): Returns ------- - the arguments with the wildcard inputs expanded + the arguments with the modified input list """ wildcard_inputs = [input_name for input_name in args.input if "*" in input_name] + file_list_inputs = [input_name for input_name in args.input if "file_list" in input_name] + for wildcard_input in wildcard_inputs: - if not glob.glob(wildcard_input): - raise FileNotFoundError( - f"Invalid wildcard input {wildcard_input}. " - f"Please ensure the wildcard pattern matches at least one file or directory." - ) - input_files = Path(".").glob(wildcard_input) - for input_file in input_files: - if input_file.is_file(): - args.input.append(str(input_file)) - elif input_file.is_dir(): - files = input_file.glob("*") - inputs = [str(file) for file in files if file.is_file() and "file_list" not in file.name] - args.input.extend(inputs) - else: - raise FileNotFoundError(f"Invalid wildcard input {wildcard_input}.") + input_files = [str(file) for file in Path(".").glob(wildcard_input)] + args.input.extend(input_files) args.input.remove(wildcard_input) - return args - -def expand_list_file(args): - """ - Expands the list of inputs by adding files from file lists and removing the file list. - - Parameters - ---------- - args argparse.Namespace - the arguments from the parser - - Returns - ------- - the arguments with the modified input list - - """ - file_list_inputs = [input_name for input_name in args.input if "file_list" in input_name] for file_list_input in file_list_inputs: with open(file_list_input, "r") as f: file_inputs = [input_name.strip() for input_name in f.readlines()] @@ -106,6 +77,7 @@ def set_input_lists(args): """ input_paths = [] + args = _expand_user_input(args) for input_name in args.input: input_path = Path(input_name).resolve() if input_path.exists(): From 0124e43071329e43b027abe9b385297edb5b1683 Mon Sep 17 00:00:00 2001 From: yucongalicechen Date: Wed, 15 May 2024 19:12:56 -0400 Subject: [PATCH 5/7] removed more unnecessary tests and added test for file-list file, currently failing --- src/diffpy/labpdfproc/labpdfprocapp.py | 4 ++-- src/diffpy/labpdfproc/tests/test_tools.py | 16 ++-------------- src/diffpy/labpdfproc/tools.py | 2 -- 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/src/diffpy/labpdfproc/labpdfprocapp.py b/src/diffpy/labpdfproc/labpdfprocapp.py index 88871c6..2807fc8 100644 --- a/src/diffpy/labpdfproc/labpdfprocapp.py +++ b/src/diffpy/labpdfproc/labpdfprocapp.py @@ -29,8 +29,8 @@ def get_args(override_cli_inputs=None): "everything in the folder ./data), 'data/file_list.txt' (load" " the list of files contained in the text-file called " "file_list.txt that can be found in the folder ./data), " - "'data/*.chi' (load all files with extension .chi in the " - "folder ./data), 'data*' (load all files and directories starting with 'data').", + "'./*.chi', 'data/*.chi' (load all files with extension .chi in the " + "folder ./data).", ) p.add_argument( "-a", diff --git a/src/diffpy/labpdfproc/tests/test_tools.py b/src/diffpy/labpdfproc/tests/test_tools.py index 926808b..015b20d 100644 --- a/src/diffpy/labpdfproc/tests/test_tools.py +++ b/src/diffpy/labpdfproc/tests/test_tools.py @@ -65,20 +65,8 @@ ["good_data*"], ["good_data.chi", "good_data.xy", "good_data.txt"], ), - ( # wildcard pattern, matching files or directories starting with input - ["input*"], - [ - "input_dir/good_data.chi", - "input_dir/good_data.xy", - "input_dir/good_data.txt", - "input_dir/unreadable_file.txt", - "input_dir/binary.pkl", - ], - ), - ( # wildcard pattern, matching files or directories starting with unreadable and ending with .txt extension - ["unreadable*.txt"], - ["unreadable_file.txt"], - ), + # wildcard pattern for file-list file, currently failing + # (["input_dir/file_list*"], "file_list.txt"), ] diff --git a/src/diffpy/labpdfproc/tools.py b/src/diffpy/labpdfproc/tools.py index cc59c07..2b5ff24 100644 --- a/src/diffpy/labpdfproc/tools.py +++ b/src/diffpy/labpdfproc/tools.py @@ -44,12 +44,10 @@ def _expand_user_input(args): """ wildcard_inputs = [input_name for input_name in args.input if "*" in input_name] file_list_inputs = [input_name for input_name in args.input if "file_list" in input_name] - for wildcard_input in wildcard_inputs: input_files = [str(file) for file in Path(".").glob(wildcard_input)] args.input.extend(input_files) args.input.remove(wildcard_input) - for file_list_input in file_list_inputs: with open(file_list_input, "r") as f: file_inputs = [input_name.strip() for input_name in f.readlines()] From 2fe62861ed3bab35960480dafa98fc846aa8bb5f Mon Sep 17 00:00:00 2001 From: yucongalicechen Date: Wed, 15 May 2024 20:47:12 -0400 Subject: [PATCH 6/7] added test case of a wildcard in a file-list file --- src/diffpy/labpdfproc/tests/conftest.py | 2 ++ src/diffpy/labpdfproc/tests/test_tools.py | 4 ++-- src/diffpy/labpdfproc/tools.py | 10 +++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/diffpy/labpdfproc/tests/conftest.py b/src/diffpy/labpdfproc/tests/conftest.py index 5b8c323..a5fe4b0 100644 --- a/src/diffpy/labpdfproc/tests/conftest.py +++ b/src/diffpy/labpdfproc/tests/conftest.py @@ -36,6 +36,8 @@ def user_filesystem(tmp_path): with open(input_dir / "binary.pkl", "wb") as f: f.write(binary_data) + with open(base_dir / "file_list.txt", "w") as f: + f.write("./*.txt") with open(input_dir / "file_list.txt", "w") as f: f.write("good_data.chi \n good_data.xy \n good_data.txt \n missing_file.txt") with open(input_dir / "file_list_example2.txt", "w") as f: diff --git a/src/diffpy/labpdfproc/tests/test_tools.py b/src/diffpy/labpdfproc/tests/test_tools.py index 015b20d..a6ecce4 100644 --- a/src/diffpy/labpdfproc/tests/test_tools.py +++ b/src/diffpy/labpdfproc/tests/test_tools.py @@ -65,8 +65,8 @@ ["good_data*"], ["good_data.chi", "good_data.xy", "good_data.txt"], ), - # wildcard pattern for file-list file, currently failing - # (["input_dir/file_list*"], "file_list.txt"), + # wildcard pattern in a file-list file + (["file_list.txt"], ["good_data.txt", "unreadable_file.txt"]), ] diff --git a/src/diffpy/labpdfproc/tools.py b/src/diffpy/labpdfproc/tools.py index 2b5ff24..ea80ec9 100644 --- a/src/diffpy/labpdfproc/tools.py +++ b/src/diffpy/labpdfproc/tools.py @@ -42,17 +42,17 @@ def _expand_user_input(args): the arguments with the modified input list """ - wildcard_inputs = [input_name for input_name in args.input if "*" in input_name] file_list_inputs = [input_name for input_name in args.input if "file_list" in input_name] - for wildcard_input in wildcard_inputs: - input_files = [str(file) for file in Path(".").glob(wildcard_input)] - args.input.extend(input_files) - args.input.remove(wildcard_input) for file_list_input in file_list_inputs: with open(file_list_input, "r") as f: file_inputs = [input_name.strip() for input_name in f.readlines()] args.input.extend(file_inputs) args.input.remove(file_list_input) + wildcard_inputs = [input_name for input_name in args.input if "*" in input_name] + for wildcard_input in wildcard_inputs: + input_files = [str(file) for file in Path(".").glob(wildcard_input) if "file_list" not in file.name] + args.input.extend(input_files) + args.input.remove(wildcard_input) return args From bfa3d8e1bf7ddc34cd1cfe2a8afe0270ab762bda Mon Sep 17 00:00:00 2001 From: yucongalicechen Date: Wed, 15 May 2024 23:28:27 -0400 Subject: [PATCH 7/7] edited wildcard in file-list file --- src/diffpy/labpdfproc/tests/conftest.py | 3 +-- src/diffpy/labpdfproc/tests/test_tools.py | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/diffpy/labpdfproc/tests/conftest.py b/src/diffpy/labpdfproc/tests/conftest.py index a5fe4b0..717d148 100644 --- a/src/diffpy/labpdfproc/tests/conftest.py +++ b/src/diffpy/labpdfproc/tests/conftest.py @@ -36,11 +36,10 @@ def user_filesystem(tmp_path): with open(input_dir / "binary.pkl", "wb") as f: f.write(binary_data) - with open(base_dir / "file_list.txt", "w") as f: - f.write("./*.txt") with open(input_dir / "file_list.txt", "w") as f: f.write("good_data.chi \n good_data.xy \n good_data.txt \n missing_file.txt") with open(input_dir / "file_list_example2.txt", "w") as f: + f.write("input_dir/*.txt \n") f.write("input_dir/good_data.chi \n") f.write("good_data.xy \n") f.write(f"{str(input_dir.resolve() / 'good_data.txt')}\n") diff --git a/src/diffpy/labpdfproc/tests/test_tools.py b/src/diffpy/labpdfproc/tests/test_tools.py index a6ecce4..be036b7 100644 --- a/src/diffpy/labpdfproc/tests/test_tools.py +++ b/src/diffpy/labpdfproc/tests/test_tools.py @@ -49,9 +49,9 @@ "input_dir/binary.pkl", ], ), - ( # file_list_example2.txt list of files provided in different directories + ( # file_list_example2.txt list of files provided in different directories with wildcard ["input_dir/file_list_example2.txt"], - ["input_dir/good_data.chi", "good_data.xy", "input_dir/good_data.txt"], + ["input_dir/good_data.chi", "good_data.xy", "input_dir/good_data.txt", "input_dir/unreadable_file.txt"], ), ( # wildcard pattern, matching files with .chi extension in the same directory ["./*.chi"], @@ -65,8 +65,6 @@ ["good_data*"], ["good_data.chi", "good_data.xy", "good_data.txt"], ), - # wildcard pattern in a file-list file - (["file_list.txt"], ["good_data.txt", "unreadable_file.txt"]), ]