diff --git a/lib/dependencies/inspect-implementation.ts b/lib/dependencies/inspect-implementation.ts index 7a8212f..d86a679 100644 --- a/lib/dependencies/inspect-implementation.ts +++ b/lib/dependencies/inspect-implementation.ts @@ -6,7 +6,11 @@ import * as subProcess from './sub-process'; import { DepGraph } from '@snyk/dep-graph'; import { buildDepGraph, PartialDepTree } from './build-dep-graph'; import { FILENAMES } from '../types'; -import { EmptyManifestError, RequiredPackagesMissingError } from '../errors'; +import { + EmptyManifestError, + RequiredPackagesMissingError, + UnparsableRequirementError, +} from '../errors'; const returnedTargetFile = (originalTargetFile) => { const basename = path.basename(originalTargetFile); @@ -271,6 +275,10 @@ export async function inspectInstalledDeps( throw new RequiredPackagesMissingError(errMsg); } + + if (error.indexOf('Unparsable requirement line') !== -1) { + throw new UnparsableRequirementError(error); + } } throw error; diff --git a/lib/errors.ts b/lib/errors.ts index 92fc187..822a1ab 100644 --- a/lib/errors.ts +++ b/lib/errors.ts @@ -1,6 +1,7 @@ export enum PythonPluginErrorNames { EMPTY_MANIFEST_ERROR = 'EMPTY_MANIFEST_ERROR', REQUIRED_PACKAGES_MISSING_ERROR = 'REQUIRED_PACKAGES_MISSING_ERROR', + UNPARSABLE_REQUIREMENT_ERROR = 'UNPARSABLE_REQUIREMENT_ERROR', } export class EmptyManifestError extends Error { @@ -16,3 +17,10 @@ export class RequiredPackagesMissingError extends Error { this.name = PythonPluginErrorNames.REQUIRED_PACKAGES_MISSING_ERROR; } } + +export class UnparsableRequirementError extends Error { + constructor(message: string) { + super(message); + this.name = PythonPluginErrorNames.UNPARSABLE_REQUIREMENT_ERROR; + } +} diff --git a/pysrc/constants.py b/pysrc/constants.py index 1e6c8d2..556fd36 100644 --- a/pysrc/constants.py +++ b/pysrc/constants.py @@ -23,3 +23,10 @@ def discover(cls, requirements_file_path): return cls.SETUPTOOLS return cls.PIP + +DEFAULT_OPTIONS = { + "allow_missing":False, + "dev_deps":False, + "only_provenance":False, + "allow_empty":False +} \ No newline at end of file diff --git a/pysrc/pip_resolve.py b/pysrc/pip_resolve.py index 7199ddb..145a640 100644 --- a/pysrc/pip_resolve.py +++ b/pysrc/pip_resolve.py @@ -9,7 +9,7 @@ import pipfile import codecs from operator import le, lt, gt, ge, eq, ne -from constants import DepsManager +from constants import DEFAULT_OPTIONS, DepsManager import pkg_resources @@ -349,7 +349,7 @@ def get_requirements_for_setuptools(requirements_file_path): with open(requirements_file_path, 'r') as f: setup_py_file_content = f.read() requirements_data = setup_file.parse_requirements(setup_py_file_content) - req_list = list(requirements.parse(requirements_data)) + req_list = [req for req in requirements.parse(requirements_data) if req is not None] provenance = setup_file.get_provenance(setup_py_file_content) for req in req_list: @@ -364,7 +364,7 @@ def get_requirements_for_setuptools(requirements_file_path): return req_list -def get_requirements_for_pip(requirements_file_path): +def get_requirements_for_pip(requirements_file_path, options): """Get requirements for a pip project. Note: @@ -381,7 +381,7 @@ def get_requirements_for_pip(requirements_file_path): encoding = detect_encoding_by_bom(requirements_file_path) with io.open(requirements_file_path, 'r', encoding=encoding) as f: - req_list = list(requirements.parse(f)) + req_list = list(requirements.parse(f, options)) req_list = filter_requirements(req_list) @@ -409,7 +409,7 @@ def filter_requirements(req_list): return req_list -def get_requirements_list(requirements_file_path, dev_deps=False): +def get_requirements_list(requirements_file_path, options=DEFAULT_OPTIONS): """Retrieves the requirements from the requirements file The requirements can be retrieved from requirements.txt, Pipfile or setup.py @@ -423,11 +423,11 @@ def get_requirements_list(requirements_file_path, dev_deps=False): empty list: if no requirements were found in the requirements file. """ if deps_manager is DepsManager.PIPENV: - req_list = get_requirements_for_pipenv(requirements_file_path, dev_deps) + req_list = get_requirements_for_pipenv(requirements_file_path, options.dev_deps) elif deps_manager is DepsManager.SETUPTOOLS: req_list = get_requirements_for_setuptools(requirements_file_path) else: - req_list = get_requirements_for_pip(requirements_file_path) + req_list = get_requirements_for_pip(requirements_file_path, options) return req_list @@ -441,10 +441,7 @@ def canonicalize_package_name(name): def create_dependencies_tree_by_req_file_path( requirements_file_path, - allow_missing=False, - dev_deps=False, - only_provenance=False, - allow_empty=False + options=DEFAULT_OPTIONS, ): # TODO: normalise package names before any other processing - this should # help reduce the amount of `in place` conversions. @@ -463,7 +460,7 @@ def create_dependencies_tree_by_req_file_path( dist_tree = utils.construct_tree(dist_index) # create a list of dependencies from the dependencies file - required = get_requirements_list(requirements_file_path, dev_deps=dev_deps) + required = get_requirements_list(requirements_file_path, options) # Handle optional dependencies/arbitrary dependencies optional_dependencies = utils.establish_optional_dependencies( @@ -475,7 +472,7 @@ def create_dependencies_tree_by_req_file_path( top_level_provenance_map = {} - if not required and not allow_empty: + if not required and not options.allow_empty: msg = 'No dependencies detected in manifest.' sys.exit(msg) else: @@ -490,7 +487,7 @@ def create_dependencies_tree_by_req_file_path( top_level_provenance_map[canonicalize_package_name(r.name)] = r.original_name if missing_package_names: msg = 'Required packages missing: ' + (', '.join(missing_package_names)) - if allow_missing: + if options.allow_missing: sys.stderr.write(msg + "\n") else: sys.exit(msg) @@ -501,8 +498,8 @@ def create_dependencies_tree_by_req_file_path( top_level_requirements, optional_dependencies, requirements_file_path, - allow_missing, - only_provenance, + options.allow_missing, + options.only_provenance, top_level_provenance_map, ) @@ -542,10 +539,7 @@ def main(): create_dependencies_tree_by_req_file_path( args.requirements, - allow_missing=args.allow_missing, - dev_deps=args.dev_deps, - only_provenance=args.only_provenance, - allow_empty=args.allow_empty, + args, ) diff --git a/pysrc/requirements/parser.py b/pysrc/requirements/parser.py index 381432b..5e64bf8 100644 --- a/pysrc/requirements/parser.py +++ b/pysrc/requirements/parser.py @@ -2,11 +2,16 @@ import os import warnings -import re +import sys from .requirement import Requirement -def parse(req_str_or_file): +def parse(req_str_or_file, options={ + "allow_missing":False, + "dev_deps":False, + "only_provenance":False, + "allow_empty":False +}): """ Parse a requirements file into a list of Requirements @@ -59,7 +64,7 @@ def parse(req_str_or_file): new_file_path = os.path.join(os.path.dirname(filename or '.'), new_filename) with open(new_file_path) as f: - for requirement in parse(f): + for requirement in parse(f, options): yield requirement elif line.startswith('-f') or line.startswith('--find-links') or \ line.startswith('-i') or line.startswith('--index-url') or \ @@ -75,7 +80,13 @@ def parse(req_str_or_file): line.split()[0]) continue else: - req = Requirement.parse(line) + try: + req = Requirement.parse(line) + except Exception as e: + if options.allow_missing: + warnings.warn("Skipping line (%s).\n Couldn't process: (%s)." %(line.split()[0], e)) + continue + sys.exit('Unparsable requirement line (%s)' %(e)) req.provenance = ( filename, original_line_idxs[0] + 1, diff --git a/test/system/inspect.spec.ts b/test/system/inspect.spec.ts index 29d35b9..64e8c43 100644 --- a/test/system/inspect.spec.ts +++ b/test/system/inspect.spec.ts @@ -740,6 +740,31 @@ describe('inspect', () => { async () => await inspect('.', FILENAMES.pip.manifest) ).rejects.toThrow('Required packages missing: markupsafe'); }); + + it('should fail on nonexistent referenced local depedency', async () => { + const workspace = 'pip-app-local-nonexistent-file'; + testUtils.chdirWorkspaces(workspace); + testUtils.ensureVirtualenv(workspace); + tearDown = testUtils.activateVirtualenv(workspace); + + await expect(inspect('.', FILENAMES.pip.manifest)).rejects.toThrow( + "Unparsable requirement line ([Errno 2] No such file or directory: './lib/nonexistent/setup.py')" + ); + }); + + it('should not fail on nonexistent referenced local depedency when --skip-unresolved', async () => { + const workspace = 'pip-app-local-nonexistent-file'; + testUtils.chdirWorkspaces(workspace); + testUtils.ensureVirtualenv(workspace); + tearDown = testUtils.activateVirtualenv(workspace); + + const result = await inspect('.', FILENAMES.pip.manifest, { + allowMissing: true, + allowEmpty: true, + }); + + expect(result.dependencyGraph.toJSON()).not.toEqual({}); + }); }); describe('Circular deps', () => { diff --git a/test/workspaces/pip-app-local-nonexistent-file/lib/nonexistent/not-setup-file.txt b/test/workspaces/pip-app-local-nonexistent-file/lib/nonexistent/not-setup-file.txt new file mode 100644 index 0000000..7c5de4b --- /dev/null +++ b/test/workspaces/pip-app-local-nonexistent-file/lib/nonexistent/not-setup-file.txt @@ -0,0 +1 @@ +This is a dummy file \ No newline at end of file diff --git a/test/workspaces/pip-app-local-nonexistent-file/requirements.txt b/test/workspaces/pip-app-local-nonexistent-file/requirements.txt new file mode 100644 index 0000000..865abff --- /dev/null +++ b/test/workspaces/pip-app-local-nonexistent-file/requirements.txt @@ -0,0 +1 @@ +./lib/nonexistent \ No newline at end of file