Skip to content

Coderbotics: Refactoring of refactor_list_test_packages.py #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Coderbotics: Refactoring of refactor_list_test_packages.py #2

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
49086a6
Refactor `process_command_line` to `process_command_line`
CoderboticsAI Jul 14, 2023
6e01ff3
Refactor `parse_package_list` to `parse_package_list`
CoderboticsAI Jul 14, 2023
ba18d98
Refactor `create_test_packages_list` to `parse_package_list`, `create…
CoderboticsAI Jul 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
214 changes: 125 additions & 89 deletions scripts/list_test_packages.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,126 +9,133 @@
from typing import Any, Dict, List

from test_packages_support import get_platform_list_path
import argparse
from typing import List


def process_command_line(argv: List[str]) -> argparse.Namespace:
"""Process command line invocation arguments and switches.

Args:
argv: list of arguments, or `None` from ``sys.argv[1:]``.

Returns:
argparse.Namespace: named attributes of arguments and switches
def parse_package_list(package_list_file: Path) -> List[Dict[str, Any]]:
"""
# script_name = argv[0]
argv = argv[1:]

# initialize the parser object:
parser = argparse.ArgumentParser(
description="Create list of needed test packages for pipx tests and local pypiserver."
)
Parse the primary package list file and return a list of dictionaries.

# specifying nargs= puts outputs of parser in list (even if nargs=1)
Each dictionary represents a package specification and may include the 'spec' key and the 'no-deps' key.

# required arguments
parser.add_argument(
"primary_package_list",
help="Main packages to examine, getting list of "
"matching distribution files and dependencies.",
)
parser.add_argument(
"package_list_dir", help="Directory to output package distribution lists."
)

# switches/options:
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="Maximum verbosity, especially for pip operations.",
)
Args:
package_list_file (Path): The path to the primary package list file to be parsed.

args = parser.parse_args(argv)
Returns:
List[Dict[str, Any]]: A list of dictionaries representing each package specification.

return args
Examples:
>>> package_list_file = Path("package_list.txt")
>>> parse_package_list(package_list_file)
[{'spec': 'package1'}, {'spec': 'package2', 'no-deps': False}]
"""
output_list: List[Dict[str, Any]] = []

def parse_line(line: str) -> Dict[str, Any]:
line_parsed = re.sub(r"#.+$", "", line)
if not re.search(r"\S", line_parsed):
raise ValueError("No valid content found in line")
line_list = line_parsed.strip().split()
if len(line_list) == 1:
return {"spec": line_list[0]}
if len(line_list) == 2:
return {
"spec": line_list[0],
"no-deps": line_list[1].lower() == "true",
}
raise ValueError("Invalid number of fields in line")

def parse_package_list(package_list_file: Path) -> List[Dict[str, Any]]:
output_list: List[Dict[str, Any]] = []
try:
with package_list_file.open("r") as package_list_fh:
for line in package_list_fh:
line_parsed = re.sub(r"#.+$", "", line)
if not re.search(r"\S", line_parsed):
continue
line_list = line_parsed.strip().split()
if len(line_list) == 1:
output_list.append({"spec": line_list[0]})
elif len(line_list) == 2:
output_list.append(
{
"spec": line_list[0],
"no-deps": line_list[1].lower() == "true",
}
)
else:
print(
f"ERROR: Unable to parse primary package list line:\n {line.strip()}"
)
try:
output_list.append(parse_line(line))
except ValueError as e:
print(f"ERROR: {e}\n {line.strip()}")
return []
except IOError:
print("ERROR: File problem reading primary package list.")
return []

return output_list


def create_test_packages_list(
package_list_dir_path: Path, primary_package_list_path: Path, verbose: bool
) -> int:
"""Create a list of test packages.

Args:
package_list_dir_path (Path): The path to the directory where the package list will be saved.
primary_package_list_path (Path): The path to the primary package list file.
verbose (bool): Whether to print verbose output or not.

Returns:
int: The exit code.

Raises:
ValueError: If there is a problem reading the primary package list.

Examples:
>>> package_list_dir_path = Path("package_list")
>>> primary_package_list_path = Path("primary_package_list.txt")
>>> verbose = True
>>> create_test_packages_list(package_list_dir_path, primary_package_list_path, verbose)
Examined package1
Examined package2 (no-deps)
"""
exit_code = 0

def get_verbose_string(verbose: bool, verbose_this_iteration: bool) -> str:
if verbose or verbose_this_iteration:
return f"\n{pip_download_process.stdout.strip()}\n{pip_download_process.stderr.strip()}"
return ""

def parse_test_package(spec: str, no_deps: bool) -> Tuple[List[str], bool]:
verbose_this_iteration = False
cmd_list = [
"pip",
"download",
*(["--no-deps"] if no_deps else []),
spec,
"-d",
str(download_dir),
]
if verbose:
print(f"CMD: {' '.join(cmd_list)}")
pip_download_process = subprocess.run(
cmd_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
if pip_download_process.returncode == 0:
print(f"Examined {spec}{' (no-deps)' if no_deps else ''}")
else:
print(
f"ERROR with {spec}{' (no-deps)' if no_deps else ''}",
file=sys.stderr,
)
verbose_this_iteration = True
exit_code = 1
return get_verbose_string(verbose, verbose_this_iteration), exit_code

package_list_dir_path.mkdir(exist_ok=True)
platform_package_list_path = get_platform_list_path(package_list_dir_path)

primary_test_packages = parse_package_list(primary_package_list_path)
if not primary_test_packages:
print(
f"ERROR: Problem reading {primary_package_list_path}. Exiting.",
file=sys.stderr,
)
return 1
raise ValueError(f"Problem reading {primary_package_list_path}.")

with tempfile.TemporaryDirectory() as download_dir:
for test_package in primary_test_packages:
test_package_option_string = (
" (no-deps)" if test_package.get("no-deps", False) else ""
)
verbose_this_iteration = False
cmd_list = (
["pip", "download"]
+ (["--no-deps"] if test_package.get("no-deps", False) else [])
+ [test_package["spec"], "-d", str(download_dir)]
)
if verbose:
print(f"CMD: {' '.join(cmd_list)}")
pip_download_process = subprocess.run(
cmd_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
verbose_string, exit_code = parse_test_package(
test_package["spec"], test_package.get("no-deps", False)
)
if pip_download_process.returncode == 0:
print(f"Examined {test_package['spec']}{test_package_option_string}")
else:
print(
f"ERROR with {test_package['spec']}{test_package_option_string}",
file=sys.stderr,
)
verbose_this_iteration = True
exit_code = 1
if verbose or verbose_this_iteration:
print(pip_download_process.stdout)
print(pip_download_process.stderr)
downloaded_list = os.listdir(download_dir)
print(verbose_string)

downloaded_list = os.listdir(download_dir)

all_packages = []
for downloaded_filename in downloaded_list:
Expand All @@ -149,13 +156,42 @@ def create_test_packages_list(
all_packages.append(f"{package_name}=={package_version}")

with platform_package_list_path.open("w") as package_list_fh:
"scripts/list_test_packages.py",
for package in sorted(all_packages):
print(package, file=package_list_fh)

return exit_code


def process_command_line(argv: List[str]) -> argparse.Namespace:
"""
Process command line invocation arguments and switches.

Args:
argv (List[str]): list of arguments, or `None` from `sys.argv[1:]`.

Returns:
argparse.Namespace: named attributes of arguments and switches
"""
parser = argparse.ArgumentParser(
description="Create a list of needed test packages for pipx tests and local pypiserver."
)
parser.add_argument(
"primary_package_list",
help="Main packages to examine, getting a list of "
"matching distribution files and dependencies.",
)
parser.add_argument(
"package_list_dir", help="Directory to output package distribution lists."
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="Maximize verbosity, especially for pip operations.",
)
return parser.parse_args(argv[1:])


def main(argv: List[str]) -> int:
args = process_command_line(argv)

Expand Down