From f4f9970e0faf3337703c160c4c8c6fd4ed2b2109 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Tue, 4 Oct 2022 15:13:21 +0300 Subject: [PATCH 01/24] New altsearch branch. --altsearch=VALUE parameter to find packages first in altsearch repo and then if not found in another repos --- crosspm/cpm.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crosspm/cpm.py b/crosspm/cpm.py index 35bf4ea..f797a94 100644 --- a/crosspm/cpm.py +++ b/crosspm/cpm.py @@ -34,6 +34,7 @@ --recursive=VALUE Process all packages recursively to find and lock all dependencies --prefer-local Do not search package if exist in cache --stdout Print info and debug message to STDOUT, error to STDERR. Otherwise - all messages to STDERR + --altsearch=VALUE First search artifact in current branch repo then if not found in branch from config """ # noqa @@ -84,6 +85,8 @@ def wrapper(self, *args, **kwargs): class CrossPM: _ready = False + altsearch = False + altsearch_branch = '' def __init__(self, args=None, throw_exceptions=None, return_result=False): self._config = None @@ -123,6 +126,11 @@ def __init__(self, args=None, throw_exceptions=None, return_result=False): else: raise Exception("Unknown value to --recursive: {}".format(recursive_str)) + if self._args['--altsearch']: + self.altsearch = True + self.altsearch_branch = self._args['--altsearch'] + self._log.info("Alternative search is on. Priority searching in [%s] branch", self.altsearch_branch) + if isinstance(self._args, str): if self._throw_exceptions: print(app_name) From adb02fd128052873c56d6042a2106e9e13a45edf Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Tue, 4 Oct 2022 15:21:47 +0300 Subject: [PATCH 02/24] fix formatting --- crosspm/cpm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crosspm/cpm.py b/crosspm/cpm.py index f797a94..07ffea9 100644 --- a/crosspm/cpm.py +++ b/crosspm/cpm.py @@ -127,9 +127,9 @@ def __init__(self, args=None, throw_exceptions=None, return_result=False): raise Exception("Unknown value to --recursive: {}".format(recursive_str)) if self._args['--altsearch']: - self.altsearch = True - self.altsearch_branch = self._args['--altsearch'] - self._log.info("Alternative search is on. Priority searching in [%s] branch", self.altsearch_branch) + altsearch = True + altsearch_branch = self._args['--altsearch'] + self._log.info("Alternative search is on. Priority searching in [%s] branch", self.altsearch_branch) if isinstance(self._args, str): if self._throw_exceptions: From 3f83f35914792ff64fd89ae23b7d02eee940ecb1 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Tue, 4 Oct 2022 15:24:43 +0300 Subject: [PATCH 03/24] zaputalsya with formatting. fix --- crosspm/cpm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crosspm/cpm.py b/crosspm/cpm.py index 07ffea9..ddcf1b0 100644 --- a/crosspm/cpm.py +++ b/crosspm/cpm.py @@ -127,8 +127,8 @@ def __init__(self, args=None, throw_exceptions=None, return_result=False): raise Exception("Unknown value to --recursive: {}".format(recursive_str)) if self._args['--altsearch']: - altsearch = True - altsearch_branch = self._args['--altsearch'] + self.altsearch = True + self.altsearch_branch = self._args['--altsearch'] self._log.info("Alternative search is on. Priority searching in [%s] branch", self.altsearch_branch) if isinstance(self._args, str): From ffc31a2351487fbbd2bb6df786c4d9703ae97458 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Fri, 7 Oct 2022 18:25:20 +0300 Subject: [PATCH 04/24] dig altsearch and altsearchbranch to chose funcs --- crosspm/adapters/artifactoryaql.py | 6 ++++++ crosspm/cpm.py | 3 ++- crosspm/helpers/downloader.py | 10 ++++++++-- crosspm/helpers/locker.py | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index a9d8471..b548236 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -57,6 +57,12 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v msg ) + if downloader.altsearch: + self._log.info('Alternative search is on. Packages will initially be searched in {} branch'.format(downloader.altsearchbranch)) + else: + self._log.info('Alternative search is off.') + + if 'verify' in source.args: _art_auth_etc['verify'] = source.args['verify'].lower in ['true', 'yes', '1'] else: diff --git a/crosspm/cpm.py b/crosspm/cpm.py index ddcf1b0..d9617c7 100644 --- a/crosspm/cpm.py +++ b/crosspm/cpm.py @@ -354,6 +354,7 @@ def command(self, command_): 'out_format': ['--out-format', ''], 'output': ['--output', ''], 'output_template': ['--output-template', ''], + 'altsearch': ['--altsearch',''], # 'out_prefix': ['--out-prefix', ''], # 'depslock_path': ['--depslock-path', ''], } @@ -386,7 +387,7 @@ def command(self, command_): if command_ is Locker: do_load = self.recursive - cpm_ = command_(self._config, do_load, self.recursive) + cpm_ = command_(self._config, do_load, self.recursive, self.altsearch, self.altsearch_branch) cpm_.entrypoint() if self._return_result: diff --git a/crosspm/helpers/downloader.py b/crosspm/helpers/downloader.py index ef1cec4..00f3c0e 100644 --- a/crosspm/helpers/downloader.py +++ b/crosspm/helpers/downloader.py @@ -16,7 +16,9 @@ def entrypoint(self, *args, **kwargs): class Downloader(Command): - def __init__(self, config, do_load, recursive): + altsearch = False + altsearchbranch = '' + def __init__(self, config, do_load, recursive, altsearch, altsearchbranch): self._log = logging.getLogger('crosspm') self._config = config # type: Config self.cache = config.cache @@ -25,6 +27,10 @@ def __init__(self, config, do_load, recursive): self._root_package = Package('', 0, {self._config.name_column: ''}, self, None, self.common_parser) self.recursive = recursive + if altsearch: + self.altsearch = altsearch + self.altsearchbranch = altsearchbranch + if not config.deps_path: config.deps_path = \ @@ -161,7 +167,7 @@ def entrypoint(self, *args, **kwargs): def search_dependencies(self, depslock_file_path, deps_content=None): self._log.info('Check dependencies ...') - self._root_package.find_dependencies(depslock_file_path, property_validate=True, deps_content=deps_content, ) + self._root_package.find_dependencies(depslock_file_path, property_validate=True, deps_content=deps_content,) self._log.info('') self.set_duplicated_flag() self._log.info('Dependency tree:') diff --git a/crosspm/helpers/locker.py b/crosspm/helpers/locker.py index 3509248..d04d25e 100644 --- a/crosspm/helpers/locker.py +++ b/crosspm/helpers/locker.py @@ -10,7 +10,7 @@ class Locker(Downloader): def __init__(self, config, do_load, recursive): # TODO: revise logic to allow recursive search without downloading - super(Locker, self).__init__(config, do_load, recursive) + super(Locker, self).__init__(config, do_load, recursive, self.altsearch, self.altsearchbranch) if not getattr(config, 'deps_path', ''): config.deps_path = config.deps_file_name or CROSSPM_DEPENDENCY_FILENAME From 13b9d88215f232cbcce24809cc30535c06ac5f76 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Fri, 7 Oct 2022 18:30:23 +0300 Subject: [PATCH 05/24] fix formatting --- crosspm/adapters/artifactoryaql.py | 4 ++-- crosspm/helpers/downloader.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index b548236..243a5db 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -58,11 +58,11 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v ) if downloader.altsearch: - self._log.info('Alternative search is on. Packages will initially be searched in {} branch'.format(downloader.altsearchbranch)) + msg = 'Alternative search is on. Packages will initially be searched in {0} branch' + self._log.info(msg.format(downloader.altsearchbranch)) else: self._log.info('Alternative search is off.') - if 'verify' in source.args: _art_auth_etc['verify'] = source.args['verify'].lower in ['true', 'yes', '1'] else: diff --git a/crosspm/helpers/downloader.py b/crosspm/helpers/downloader.py index 00f3c0e..c337db4 100644 --- a/crosspm/helpers/downloader.py +++ b/crosspm/helpers/downloader.py @@ -18,6 +18,7 @@ def entrypoint(self, *args, **kwargs): class Downloader(Command): altsearch = False altsearchbranch = '' + def __init__(self, config, do_load, recursive, altsearch, altsearchbranch): self._log = logging.getLogger('crosspm') self._config = config # type: Config @@ -28,9 +29,8 @@ def __init__(self, config, do_load, recursive, altsearch, altsearchbranch): self.common_parser) self.recursive = recursive if altsearch: - self.altsearch = altsearch - self.altsearchbranch = altsearchbranch - + self.altsearch = altsearch + self.altsearchbranch = altsearchbranch if not config.deps_path: config.deps_path = \ From de892aa6c88f47a57b5042cf200625b5bd920690 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Fri, 7 Oct 2022 18:31:58 +0300 Subject: [PATCH 06/24] fix formatting vol. 2 --- crosspm/cpm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crosspm/cpm.py b/crosspm/cpm.py index d9617c7..15bdb49 100644 --- a/crosspm/cpm.py +++ b/crosspm/cpm.py @@ -354,7 +354,7 @@ def command(self, command_): 'out_format': ['--out-format', ''], 'output': ['--output', ''], 'output_template': ['--output-template', ''], - 'altsearch': ['--altsearch',''], + 'altsearch': ['--altsearch', ''], # 'out_prefix': ['--out-prefix', ''], # 'depslock_path': ['--depslock-path', ''], } From 0fb067a9053b5c897022414ba9478bc9c6dce2be Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Thu, 27 Oct 2022 16:47:10 +0300 Subject: [PATCH 07/24] fix merge problems --- crosspm/cpm.py | 2 +- crosspm/helpers/downloader.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crosspm/cpm.py b/crosspm/cpm.py index 973843a..5dd347d 100644 --- a/crosspm/cpm.py +++ b/crosspm/cpm.py @@ -378,7 +378,7 @@ def command(self, command_): if command_ is Locker: do_load = self._config.recursive - cpm_ = command_(self._config, do_load, self.altsearch, self.altsearch_branch) + cpm_ = command_(self.altsearch, self.altsearch_branch, self._config, do_load) cpm_.entrypoint() if self._return_result: diff --git a/crosspm/helpers/downloader.py b/crosspm/helpers/downloader.py index c1fa700..ae2c9ff 100644 --- a/crosspm/helpers/downloader.py +++ b/crosspm/helpers/downloader.py @@ -19,7 +19,7 @@ class Downloader(Command): altsearch = False altsearchbranch = '' - def __init__(self,altsearch, altsearchbranch, config: Config, do_load: bool, recursive: Optional[bool] = None): + def __init__(self, altsearch, altsearchbranch, config: Config, do_load: bool, recursive: Optional[bool] = None): self._log = logging.getLogger('crosspm') self._config = config # type: Config self.cache = config.cache From 2585f02369d6047c6fb11bab74f3903553999c0e Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Thu, 27 Oct 2022 16:50:11 +0300 Subject: [PATCH 08/24] fix formatting issues --- crosspm/helpers/downloader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crosspm/helpers/downloader.py b/crosspm/helpers/downloader.py index ae2c9ff..17c8adb 100644 --- a/crosspm/helpers/downloader.py +++ b/crosspm/helpers/downloader.py @@ -19,6 +19,7 @@ class Downloader(Command): altsearch = False altsearchbranch = '' + def __init__(self, altsearch, altsearchbranch, config: Config, do_load: bool, recursive: Optional[bool] = None): self._log = logging.getLogger('crosspm') self._config = config # type: Config @@ -31,7 +32,7 @@ def __init__(self, altsearch, altsearchbranch, config: Config, do_load: bool, re self.recursive = config.recursive if recursive is None else recursive if altsearch: self.altsearch = altsearch - self.altsearchbranch = altsearchbranch + self.altsearchbranch = altsearchbranch self.do_load = do_load From 138b2f17e5817932b19979d56f92ac3a82a24f71 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Thu, 27 Oct 2022 17:25:43 +0300 Subject: [PATCH 09/24] fix parameters --- crosspm/helpers/locker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crosspm/helpers/locker.py b/crosspm/helpers/locker.py index bf039a9..15296e7 100644 --- a/crosspm/helpers/locker.py +++ b/crosspm/helpers/locker.py @@ -9,7 +9,7 @@ class Locker(Downloader): def __init__(self, config: Config, do_load: bool, recursive: Optional[bool] = None): # TODO: revise logic to allow recursive search without downloading - super(Locker, self).__init__(config, do_load, recursive, self.altsearch, self.altsearchbranch) + super(Locker, self).__init__(self.altsearch, self.altsearchbranch, config, do_load, recursive) def lock_packages(self, packages: Optional[Dict[str, Package]] = None): """ From 56358f06eefe076d1292bc5b46dc162dd5e10471 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Thu, 27 Oct 2022 17:59:48 +0300 Subject: [PATCH 10/24] get alt paths list --- crosspm/adapters/artifactoryaql.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index 4f268af..29d7624 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -31,6 +31,19 @@ class Adapter(BaseAdapter): + + def get_alt_paths_list(self, list_or_file_path, downloader): + list_or_file_path_alt = copy.deepcopy(list_or_file_path) + + for element in list_or_file_path_alt["raw"]: + if element['altsearch'] and element['altsearch'] == '+': + element['branch'] = downloader.altsearchbranch + + for element in list_or_file_path['raw']: + if not (element['altsearch'] and element['altsearch'] == '+'): + list_or_file_path_alt['raw'].remove(element) + return list_or_file_path_alt + def get_packages(self, source, parser, downloader, list_or_file_path, property_validate=True): """ @@ -77,6 +90,9 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v _packed_exist = False _packed_cache_params = None self._log.info('parser: {}'.format(parser._name)) + + list_or_file_path_alt = self.get_alt_paths_list(list_or_file_path, downloader) + for _paths in parser.get_paths(list_or_file_path, source): # If "parser"-column specified - find only in this parser From 224c4d80b8d71e56ec874a4d040c431b8416ba11 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Thu, 27 Oct 2022 18:33:53 +0300 Subject: [PATCH 11/24] refactor artifactory search to separate method --- crosspm/adapters/artifactoryaql.py | 69 ++++++++++++++++-------------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index 29d7624..91033f1 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -44,6 +44,38 @@ def get_alt_paths_list(self, list_or_file_path, downloader): list_or_file_path_alt['raw'].remove(element) return list_or_file_path_alt + def artifactory_search(self, _tmp_params, _art_auth_etc, _file_name_pattern,_path_pattern, _path_fixed): + _artifactory_server = _tmp_params['server'] + _search_repo = _tmp_params['repo'] + + # Get AQL path pattern, with fixed part path, without artifactory url and repository name + _aql_path_pattern = _path_fixed[len(_artifactory_server) + 1 + len(_search_repo) + 1:] + if _path_pattern: + _aql_path_pattern = _aql_path_pattern + "/" + _path_pattern + + _aql_query_url = '{}/api/search/aql'.format(_artifactory_server) + _aql_query_dict = { + "repo": { + "$eq": _search_repo, + }, + "path": { + "$match": _aql_path_pattern, + }, + "name": { + "$match": _file_name_pattern, + }, + } + # Remove path if is empty string + if not _aql_path_pattern: + _aql_query_dict.pop('path') + query = 'items.find({query_dict}).include("*", "property")'.format( + query_dict=json.dumps(_aql_query_dict)) + session.auth = _art_auth_etc['auth'] + response = session.post(_aql_query_url, data=query, verify=_art_auth_etc['verify']) + response.raise_for_status() + + return response + def get_packages(self, source, parser, downloader, list_or_file_path, property_validate=True): """ @@ -121,6 +153,8 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v for _sub_paths in _paths['paths']: _tmp_params = dict(_paths['params']) self._log.info('repo: {}'.format(_sub_paths['repo'])) + +#обработка конкретного пути for _path in _sub_paths['paths']: _tmp_params['repo'] = _sub_paths['repo'] # ------ START ---- @@ -147,39 +181,12 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v # ------ END ---- _path_fixed, _path_pattern, _file_name_pattern = parser.split_fixed_pattern_with_file_name(_path) try: - _artifactory_server = _tmp_params['server'] - _search_repo = _tmp_params['repo'] - - # Get AQL path pattern, with fixed part path, without artifactory url and repository name - _aql_path_pattern = _path_fixed[len(_artifactory_server) + 1 + len(_search_repo) + 1:] - if _path_pattern: - _aql_path_pattern = _aql_path_pattern + "/" + _path_pattern - - _aql_query_url = '{}/api/search/aql'.format(_artifactory_server) - _aql_query_dict = { - "repo": { - "$eq": _search_repo, - }, - "path": { - "$match": _aql_path_pattern, - }, - "name": { - "$match": _file_name_pattern, - }, - } - # Remove path if is empty string - if not _aql_path_pattern: - _aql_query_dict.pop('path') - query = 'items.find({query_dict}).include("*", "property")'.format( - query_dict=json.dumps(_aql_query_dict)) - session.auth = _art_auth_etc['auth'] - r = session.post(_aql_query_url, data=query, verify=_art_auth_etc['verify']) - r.raise_for_status() - - _found_paths = r.json() + searchresults = self.artifactory_search( _tmp_params, _art_auth_etc, _file_name_pattern,_path_pattern, _path_fixed) + + _found_paths = searchresults.json() for _found in _found_paths['results']: _repo_path = "{artifactory}/{repo}/{path}/{file_name}".format( - artifactory=_artifactory_server, + artifactory=_tmp_params['server'], repo=_found['repo'], path=_found['path'], file_name=_found['name']) From 006c30623a80e52a684b78279fd4baf504ac6027 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Thu, 27 Oct 2022 18:45:34 +0300 Subject: [PATCH 12/24] fix formatting --- crosspm/adapters/artifactoryaql.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index 91033f1..a7540d9 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -44,7 +44,7 @@ def get_alt_paths_list(self, list_or_file_path, downloader): list_or_file_path_alt['raw'].remove(element) return list_or_file_path_alt - def artifactory_search(self, _tmp_params, _art_auth_etc, _file_name_pattern,_path_pattern, _path_fixed): + def artifactory_search(self, _tmp_params, _art_auth_etc, _file_name_pattern, _path_pattern, _path_fixed): _artifactory_server = _tmp_params['server'] _search_repo = _tmp_params['repo'] @@ -78,7 +78,6 @@ def artifactory_search(self, _tmp_params, _art_auth_etc, _file_name_pattern,_pat def get_packages(self, source, parser, downloader, list_or_file_path, property_validate=True): """ - :param source: :param parser: :param downloader: @@ -154,7 +153,7 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v _tmp_params = dict(_paths['params']) self._log.info('repo: {}'.format(_sub_paths['repo'])) -#обработка конкретного пути +# обработка конкретного пути for _path in _sub_paths['paths']: _tmp_params['repo'] = _sub_paths['repo'] # ------ START ---- @@ -180,8 +179,11 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v break # break connect to artifactory # ------ END ---- _path_fixed, _path_pattern, _file_name_pattern = parser.split_fixed_pattern_with_file_name(_path) + +# непосредственно, поиск в артифактории try: - searchresults = self.artifactory_search( _tmp_params, _art_auth_etc, _file_name_pattern,_path_pattern, _path_fixed) + searchresults = self.artifactory_search(_tmp_params, _art_auth_etc, + _file_name_pattern, _path_pattern, _path_fixed) _found_paths = searchresults.json() for _found in _found_paths['results']: From 3021a37cdffd205ee31dd19e06b07391a9467704 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Thu, 27 Oct 2022 18:48:08 +0300 Subject: [PATCH 13/24] fix tests --- crosspm/adapters/artifactoryaql.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index a7540d9..b469081 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -123,6 +123,10 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v self._log.info('parser: {}'.format(parser._name)) list_or_file_path_alt = self.get_alt_paths_list(list_or_file_path, downloader) +# временный цикл, чтобы тесты не ругались + for item in list_or_file_path_alt: + print(item) + for _paths in parser.get_paths(list_or_file_path, source): From e46c984a3febb0a7773ba65c451cf2b4d0ace7da Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Thu, 27 Oct 2022 18:49:15 +0300 Subject: [PATCH 14/24] fix tests --- crosspm/adapters/artifactoryaql.py | 1 - 1 file changed, 1 deletion(-) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index b469081..f8ba609 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -127,7 +127,6 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v for item in list_or_file_path_alt: print(item) - for _paths in parser.get_paths(list_or_file_path, source): # If "parser"-column specified - find only in this parser From e27aaace5824019c62e2d62d4d81f255ffd3a768 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Fri, 28 Oct 2022 10:05:27 +0300 Subject: [PATCH 15/24] fix tests --- .github/workflows/pr-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 526d68f..b7d5049 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -37,7 +37,7 @@ jobs: - name: Run Tests on Python ${{ matrix.python-version }} run: | python -mflake8 crosspm - python -mcoverage run -m py.test tests + python -mcoverage run -m pytest tests build-artifact: uses: ./.github/workflows/build.yml needs: tests From 15585f807c922b839c27f98328bce7fda639e3ea Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Mon, 31 Oct 2022 16:20:00 +0300 Subject: [PATCH 16/24] yet another commit. added altpath vars --- crosspm/adapters/artifactoryaql.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index f8ba609..e94ec9e 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -122,13 +122,13 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v _packed_cache_params = None self._log.info('parser: {}'.format(parser._name)) - list_or_file_path_alt = self.get_alt_paths_list(list_or_file_path, downloader) -# временный цикл, чтобы тесты не ругались - for item in list_or_file_path_alt: - print(item) + if downloader.altsearch: + list_or_file_path_alt = self.get_alt_paths_list(list_or_file_path, downloader) + altpaths = parser.get_paths(list_or_file_path_alt, source) - for _paths in parser.get_paths(list_or_file_path, source): + for _paths in parser.get_paths(list_or_file_path, source): + _altpath = [] # If "parser"-column specified - find only in this parser parser_names = _paths['params'].get('parser') if parser_names and parser_names != "*": @@ -143,6 +143,13 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v _params_found_raw = {} last_error = '' _pkg_name = _paths['params'][_pkg_name_column] +# выбор нужного альтернативного пути + if downloader.altsearch: + for item in altpaths: + if item['params'][_pkg_name_column] == _pkg_name: + _altpath = item + break + if _pkg_name != _pkg_name_old: _pkg_name_old = _pkg_name self._log.info( From 58eeb749a2b643b71c4bd76525e69ff81a2753cc Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Tue, 1 Nov 2022 13:45:37 +0300 Subject: [PATCH 17/24] it works --- crosspm/adapters/artifactoryaql.py | 264 +++++++++++++++-------------- 1 file changed, 134 insertions(+), 130 deletions(-) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index e94ec9e..d9c1270 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -76,6 +76,102 @@ def artifactory_search(self, _tmp_params, _art_auth_etc, _file_name_pattern, _pa return response + + def searchpackage(self, downloader, _sub_paths, parser, _paths, _art_auth_etc, property_validate): + for _path in _sub_paths['paths']: + self._tmp_params['repo'] = _sub_paths['repo'] + # ------ START ---- + # HACK for prefer-local + if self._config.prefer_local and not parser.has_rule('properties'): + params = parser.get_params_with_extra('path', _paths['params']) + for param in params: + _repo_path = ArtifactoryPath(_path, **_art_auth_etc) + param['repo'] = self._tmp_params['repo'] + param['filename'] = str(_repo_path.name) + # downloader.cache.path_any doesn't work with wildcards. + # without unnecessary checks search will be faster + if '*' in param['filename']: + self._packed_exist = False + else: + _path_packed = downloader.cache.path_packed(None, param) + self._packed_exist = os.path.isfile(_path_packed) + if self._packed_exist: + self._log.info("Skip searching, use package cache in path {}".format(_path_packed)) + _packed_cache_params = param + break # break check local cache + if self._packed_exist: + break # break connect to artifactory + # ------ END ---- + _path_fixed, _path_pattern, _file_name_pattern = parser.split_fixed_pattern_with_file_name(_path) + + # непосредственно, поиск в артифактории + try: + searchresults = self.artifactory_search(self._tmp_params, _art_auth_etc, + _file_name_pattern, _path_pattern, _path_fixed) + + _found_paths = searchresults.json() + for _found in _found_paths['results']: + _repo_path = "{artifactory}/{repo}/{path}/{file_name}".format( + artifactory=self._tmp_params['server'], + repo=_found['repo'], + path=_found['path'], + file_name=_found['name']) + _repo_path = ArtifactoryPath(_repo_path, **_art_auth_etc) + + _mark = 'found' + _matched, _params, _params_raw = parser.validate_path(str(_repo_path), self._tmp_params) + if _matched: + self._params_found[_repo_path] = {k: v for k, v in _params.items()} + self._params_found_raw[_repo_path] = {k: v for k, v in _params_raw.items()} + _mark = 'match' + + # Check if it's `root` packages or from `lock` file + # ALSO, if from `lock` and have * in name - validate with property + property_validate_tmp = property_validate or '*' in _file_name_pattern + property_found = 'properties' in _found.keys() + # If have not rule in config, skip this part + if parser.has_rule('properties') and property_validate_tmp and property_found: + _found_properties = {x['key']: x.get('value', '') for x in _found['properties']} + _valid, _params = parser.validate(_found_properties, 'properties', self._tmp_params, + return_params=True) + else: + _valid, _params = True, {} + if _valid: + _mark = 'valid' + self._packages += [_repo_path] + self._params_found[_repo_path].update({k: v for k, v in _params.items()}) + self._params_found[_repo_path]['filename'] = str(_repo_path.name) + self._params_found[_repo_path]['parser'] = parser._name + self._log.debug(' {}: {}'.format(_mark, str(_repo_path))) + except RuntimeError as e: + try: + err = json.loads(e.args[0]) + except Exception: + err = {} + if isinstance(err, dict): + # Check errors + # :e.args[0]: { + # "errors" : [ { + # "status" : 404, + # "message" : "Not Found" + # } ] + # } + for error in err.get('errors', []): + err_status = error.get('status', -1) + err_msg = error.get('message', '') + if err_status == 401: + msg = 'Authentication error[{}]{}'.format(err_status, + (': {}'.format( + err_msg)) if err_msg else '') + elif err_status == 404: + msg = last_error + else: + msg = 'Error[{}]{}'.format(err_status, + (': {}'.format(err_msg)) if err_msg else '') + if last_error != msg: + self._log.error(msg) + last_error = msg + def get_packages(self, source, parser, downloader, list_or_file_path, property_validate=True): """ :param source: @@ -118,7 +214,7 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v _secret_variables = self._config.secret_variables _packages_found = OrderedDict() _pkg_name_old = "" - _packed_exist = False + self._packed_exist = False _packed_cache_params = None self._log.info('parser: {}'.format(parser._name)) @@ -138,9 +234,9 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v self._log.info("Skip parser: {}".format(parser._name)) continue - _packages = [] - _params_found = {} - _params_found_raw = {} + self._packages = [] + self._params_found = {} + self._params_found_raw = {} last_error = '' _pkg_name = _paths['params'][_pkg_name_column] # выбор нужного альтернативного пути @@ -160,108 +256,16 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v ) ) for _sub_paths in _paths['paths']: - _tmp_params = dict(_paths['params']) + self._tmp_params = dict(_paths['params']) self._log.info('repo: {}'.format(_sub_paths['repo'])) # обработка конкретного пути - for _path in _sub_paths['paths']: - _tmp_params['repo'] = _sub_paths['repo'] - # ------ START ---- - # HACK for prefer-local - if self._config.prefer_local and not parser.has_rule('properties'): - params = parser.get_params_with_extra('path', _paths['params']) - for param in params: - _repo_path = ArtifactoryPath(_path, **_art_auth_etc) - param['repo'] = _tmp_params['repo'] - param['filename'] = str(_repo_path.name) - # downloader.cache.path_any doesn't work with wildcards. - # without unnecessary checks search will be faster - if '*' in param['filename']: - _packed_exist = False - else: - _path_packed = downloader.cache.path_packed(None, param) - _packed_exist = os.path.isfile(_path_packed) - if _packed_exist: - self._log.info("Skip searching, use package cache in path {}".format(_path_packed)) - _packed_cache_params = param - break # break check local cache - if _packed_exist: - break # break connect to artifactory - # ------ END ---- - _path_fixed, _path_pattern, _file_name_pattern = parser.split_fixed_pattern_with_file_name(_path) - -# непосредственно, поиск в артифактории - try: - searchresults = self.artifactory_search(_tmp_params, _art_auth_etc, - _file_name_pattern, _path_pattern, _path_fixed) - - _found_paths = searchresults.json() - for _found in _found_paths['results']: - _repo_path = "{artifactory}/{repo}/{path}/{file_name}".format( - artifactory=_tmp_params['server'], - repo=_found['repo'], - path=_found['path'], - file_name=_found['name']) - _repo_path = ArtifactoryPath(_repo_path, **_art_auth_etc) - - _mark = 'found' - _matched, _params, _params_raw = parser.validate_path(str(_repo_path), _tmp_params) - if _matched: - _params_found[_repo_path] = {k: v for k, v in _params.items()} - _params_found_raw[_repo_path] = {k: v for k, v in _params_raw.items()} - _mark = 'match' - - # Check if it's `root` packages or from `lock` file - # ALSO, if from `lock` and have * in name - validate with property - property_validate_tmp = property_validate or '*' in _file_name_pattern - property_found = 'properties' in _found.keys() - # If have not rule in config, skip this part - if parser.has_rule('properties') and property_validate_tmp and property_found: - _found_properties = {x['key']: x.get('value', '') for x in _found['properties']} - _valid, _params = parser.validate(_found_properties, 'properties', _tmp_params, - return_params=True) - else: - _valid, _params = True, {} - if _valid: - _mark = 'valid' - _packages += [_repo_path] - _params_found[_repo_path].update({k: v for k, v in _params.items()}) - _params_found[_repo_path]['filename'] = str(_repo_path.name) - _params_found[_repo_path]['parser'] = parser._name - self._log.debug(' {}: {}'.format(_mark, str(_repo_path))) - except RuntimeError as e: - try: - err = json.loads(e.args[0]) - except Exception: - err = {} - if isinstance(err, dict): - # Check errors - # :e.args[0]: { - # "errors" : [ { - # "status" : 404, - # "message" : "Not Found" - # } ] - # } - for error in err.get('errors', []): - err_status = error.get('status', -1) - err_msg = error.get('message', '') - if err_status == 401: - msg = 'Authentication error[{}]{}'.format(err_status, - (': {}'.format( - err_msg)) if err_msg else '') - elif err_status == 404: - msg = last_error - else: - msg = 'Error[{}]{}'.format(err_status, - (': {}'.format(err_msg)) if err_msg else '') - if last_error != msg: - self._log.error(msg) - last_error = msg + self.searchpackage(downloader, _sub_paths, parser, _paths, _art_auth_etc, property_validate) _package = None # HACK for prefer-local - if _packed_exist: + if self._packed_exist: # HACK - Normalize params for cached archive for key, value in _packed_cache_params.items(): if isinstance(value, list): @@ -270,24 +274,24 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v _package = Package(_pkg_name, None, _paths['params'], downloader, self, parser, _packed_cache_params, list_or_file_path['raw'], {}, in_cache=True) # END HACK - if _packages: - _tmp = copy.deepcopy(_params_found) - _packages = parser.filter_one(_packages, _paths['params'], _tmp) - if isinstance(_packages, dict): - _packages = [_packages] - - if len(_packages) == 1: - _stat_pkg = self.pkg_stat(_packages[0]['path']) - - _params_raw = _params_found_raw.get(_packages[0]['path'], {}) - _params_tmp = _params_found.get(_packages[0]['path'], {}) - _params_tmp.update({k: v for k, v in _packages[0]['params'].items() if k not in _params_tmp}) - _package = Package(_pkg_name, _packages[0]['path'], _paths['params'], downloader, self, parser, + if self._packages: + _tmp = copy.deepcopy(self._params_found) + self._packages = parser.filter_one(self._packages, _paths['params'], _tmp) + if isinstance(self._packages, dict): + self._packages = [self._packages] + + if len(self._packages) == 1: + _stat_pkg = self.pkg_stat(self._packages[0]['path']) + + _params_raw = self._params_found_raw.get(self._packages[0]['path'], {}) + _params_tmp = self._params_found.get(self._packages[0]['path'], {}) + _params_tmp.update({k: v for k, v in self._packages[0]['params'].items() if k not in _params_tmp}) + _package = Package(_pkg_name, self._packages[0]['path'], _paths['params'], downloader, self, parser, _params_tmp, _params_raw, _stat_pkg) _mark = 'chosen' - self._log.info(' {}: {}'.format(_mark, str(_packages[0]['path']))) + self._log.info(' {}: {}'.format(_mark, str(self._packages[0]['path']))) - elif len(_packages) > 1: + elif len(self._packages) > 1: raise CrosspmException( CROSSPM_ERRORCODE_MULTIPLE_DEPS, 'Multiple instances found for package [{}] not found.'.format(_pkg_name) @@ -424,9 +428,9 @@ def get_usedby(self, source, parser, downloader, list_or_file_path, property_val _pkg_name_old = "" for _paths in parser.get_paths(list_or_file_path, source): - _packages = [] - _params_found = {} - _params_found_raw = {} + self._packages = [] + self._params_found = {} + self._params_found_raw = {} last_error = '' _pkg_name = _paths['params'][_pkg_name_col] if _pkg_name != _pkg_name_old: @@ -439,12 +443,12 @@ def get_usedby(self, source, parser, downloader, list_or_file_path, property_val ) ) for _sub_paths in _paths['paths']: - _tmp_params = dict(_paths['params']) + self._tmp_params = dict(_paths['params']) self._log.info('repo: {}'.format(_sub_paths['repo'])) - _tmp_params['repo'] = _sub_paths['repo'] + self._tmp_params['repo'] = _sub_paths['repo'] try: - _artifactory_server = _tmp_params['server'] - _search_repo = _tmp_params['repo'] + _artifactory_server = self._tmp_params['server'] + _search_repo = self._tmp_params['repo'] # TODO: Попробовать использовать lru_cache для кеширования кучи запросов _aql_query_url = '{}/api/search/aql'.format(_artifactory_server) @@ -453,7 +457,7 @@ def get_usedby(self, source, parser, downloader, list_or_file_path, property_val "$eq": _search_repo, }, } - _usedby_aql = parser.get_usedby_aql(_tmp_params) + _usedby_aql = parser.get_usedby_aql(self._tmp_params) if _usedby_aql is None: continue _aql_query_dict.update(_usedby_aql) @@ -473,15 +477,15 @@ def get_usedby(self, source, parser, downloader, list_or_file_path, property_val _repo_path = ArtifactoryPath(_repo_path, **_art_auth_etc) _found_properties = {x['key']: x.get('value', '') for x in _found['properties']} - _matched, _params, _params_raw = parser.validate_path(str(_repo_path), _tmp_params) - _params_found[_repo_path] = {k: v for k, v in _params.items()} - _params_found_raw[_repo_path] = {k: v for k, v in _params_raw.items()} - _params = _tmp_params - _packages += [_repo_path] - _params_found[_repo_path].update({k: v for k, v in _params.items()}) - _params_found[_repo_path]['filename'] = str(_repo_path.name) + _matched, _params, _params_raw = parser.validate_path(str(_repo_path), self._tmp_params) + self._params_found[_repo_path] = {k: v for k, v in _params.items()} + self._params_found_raw[_repo_path] = {k: v for k, v in _params_raw.items()} + _params = self._tmp_params + self._packages += [_repo_path] + self._params_found[_repo_path].update({k: v for k, v in _params.items()}) + self._params_found[_repo_path]['filename'] = str(_repo_path.name) - _params_raw = _params_found_raw.get(_repo_path, {}) + _params_raw = self._params_found_raw.get(_repo_path, {}) params_found = {} # TODO: Проставление params брать из config.yaml usedby From 3025e4321aa1c76313dcace9433b6a5f19ef6039 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Tue, 1 Nov 2022 17:28:08 +0300 Subject: [PATCH 18/24] almost done. But ot yet --- crosspm/adapters/artifactoryaql.py | 214 ++++++++++++++++------------- 1 file changed, 116 insertions(+), 98 deletions(-) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index d9c1270..5f08bd2 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -77,100 +77,111 @@ def artifactory_search(self, _tmp_params, _art_auth_etc, _file_name_pattern, _pa return response - def searchpackage(self, downloader, _sub_paths, parser, _paths, _art_auth_etc, property_validate): - for _path in _sub_paths['paths']: - self._tmp_params['repo'] = _sub_paths['repo'] - # ------ START ---- - # HACK for prefer-local - if self._config.prefer_local and not parser.has_rule('properties'): - params = parser.get_params_with_extra('path', _paths['params']) - for param in params: - _repo_path = ArtifactoryPath(_path, **_art_auth_etc) - param['repo'] = self._tmp_params['repo'] - param['filename'] = str(_repo_path.name) - # downloader.cache.path_any doesn't work with wildcards. - # without unnecessary checks search will be faster - if '*' in param['filename']: - self._packed_exist = False - else: - _path_packed = downloader.cache.path_packed(None, param) - self._packed_exist = os.path.isfile(_path_packed) - if self._packed_exist: - self._log.info("Skip searching, use package cache in path {}".format(_path_packed)) - _packed_cache_params = param - break # break check local cache - if self._packed_exist: - break # break connect to artifactory - # ------ END ---- - _path_fixed, _path_pattern, _file_name_pattern = parser.split_fixed_pattern_with_file_name(_path) - - # непосредственно, поиск в артифактории - try: - searchresults = self.artifactory_search(self._tmp_params, _art_auth_etc, - _file_name_pattern, _path_pattern, _path_fixed) - - _found_paths = searchresults.json() - for _found in _found_paths['results']: - _repo_path = "{artifactory}/{repo}/{path}/{file_name}".format( - artifactory=self._tmp_params['server'], - repo=_found['repo'], - path=_found['path'], - file_name=_found['name']) - _repo_path = ArtifactoryPath(_repo_path, **_art_auth_etc) - - _mark = 'found' - _matched, _params, _params_raw = parser.validate_path(str(_repo_path), self._tmp_params) - if _matched: - self._params_found[_repo_path] = {k: v for k, v in _params.items()} - self._params_found_raw[_repo_path] = {k: v for k, v in _params_raw.items()} - _mark = 'match' - - # Check if it's `root` packages or from `lock` file - # ALSO, if from `lock` and have * in name - validate with property - property_validate_tmp = property_validate or '*' in _file_name_pattern - property_found = 'properties' in _found.keys() - # If have not rule in config, skip this part - if parser.has_rule('properties') and property_validate_tmp and property_found: - _found_properties = {x['key']: x.get('value', '') for x in _found['properties']} - _valid, _params = parser.validate(_found_properties, 'properties', self._tmp_params, - return_params=True) + def searchpackage(self, downloader, parser, _paths, _art_auth_etc, property_validate): + packagefound = False + for _sub_paths in _paths['paths']: + self._tmp_params = dict(_paths['params']) + self._log.info('repo: {}'.format(_sub_paths['repo'])) + + for _path in _sub_paths['paths']: + self._tmp_params['repo'] = _sub_paths['repo'] + # ------ START ---- + # HACK for prefer-local + if self._config.prefer_local and not parser.has_rule('properties'): + params = parser.get_params_with_extra('path', _paths['params']) + for param in params: + _repo_path = ArtifactoryPath(_path, **_art_auth_etc) + param['repo'] = self._tmp_params['repo'] + param['filename'] = str(_repo_path.name) + # downloader.cache.path_any doesn't work with wildcards. + # without unnecessary checks search will be faster + if '*' in param['filename']: + self._packed_exist = False else: - _valid, _params = True, {} - if _valid: - _mark = 'valid' - self._packages += [_repo_path] - self._params_found[_repo_path].update({k: v for k, v in _params.items()}) - self._params_found[_repo_path]['filename'] = str(_repo_path.name) - self._params_found[_repo_path]['parser'] = parser._name - self._log.debug(' {}: {}'.format(_mark, str(_repo_path))) - except RuntimeError as e: + _path_packed = downloader.cache.path_packed(None, param) + self._packed_exist = os.path.isfile(_path_packed) + if self._packed_exist: + self._log.info("Skip searching, use package cache in path {}".format(_path_packed)) + _packed_cache_params = param + break # break check local cache + if self._packed_exist: + break # break connect to artifactory + # ------ END ---- + _path_fixed, _path_pattern, _file_name_pattern = parser.split_fixed_pattern_with_file_name(_path) + + # непосредственно, поиск в артифактории try: - err = json.loads(e.args[0]) - except Exception: - err = {} - if isinstance(err, dict): - # Check errors - # :e.args[0]: { - # "errors" : [ { - # "status" : 404, - # "message" : "Not Found" - # } ] - # } - for error in err.get('errors', []): - err_status = error.get('status', -1) - err_msg = error.get('message', '') - if err_status == 401: - msg = 'Authentication error[{}]{}'.format(err_status, - (': {}'.format( - err_msg)) if err_msg else '') - elif err_status == 404: - msg = last_error - else: - msg = 'Error[{}]{}'.format(err_status, - (': {}'.format(err_msg)) if err_msg else '') - if last_error != msg: - self._log.error(msg) - last_error = msg + searchresults = self.artifactory_search(self._tmp_params, _art_auth_etc, + _file_name_pattern, _path_pattern, _path_fixed) + + _found_paths = searchresults.json() + if _found_paths['results'] != []: + packagefound = True + for _found in _found_paths['results']: + _repo_path = "{artifactory}/{repo}/{path}/{file_name}".format( + artifactory=self._tmp_params['server'], + repo=_found['repo'], + path=_found['path'], + file_name=_found['name']) + _repo_path = ArtifactoryPath(_repo_path, **_art_auth_etc) + + _mark = 'found' + _matched, _params, _params_raw = parser.validate_path(str(_repo_path), self._tmp_params) + if _matched: + self._params_found[_repo_path] = {k: v for k, v in _params.items()} + self._params_found_raw[_repo_path] = {k: v for k, v in _params_raw.items()} + _mark = 'match' + + # Check if it's `root` packages or from `lock` file + # ALSO, if from `lock` and have * in name - validate with property + property_validate_tmp = property_validate or '*' in _file_name_pattern + property_found = 'properties' in _found.keys() + # If have not rule in config, skip this part + if parser.has_rule('properties') and property_validate_tmp and property_found: + _found_properties = {x['key']: x.get('value', '') for x in _found['properties']} + _valid, _params = parser.validate(_found_properties, 'properties', self._tmp_params, + return_params=True) + else: + _valid, _params = True, {} + if _valid: + _mark = 'valid' + self._packages += [_repo_path] + self._params_found[_repo_path].update({k: v for k, v in _params.items()}) + self._params_found[_repo_path]['filename'] = str(_repo_path.name) + self._params_found[_repo_path]['parser'] = parser._name + self._log.debug(' {}: {}'.format(_mark, str(_repo_path))) + except RuntimeError as e: + try: + err = json.loads(e.args[0]) + except Exception: + err = {} + if isinstance(err, dict): + # Check errors + # :e.args[0]: { + # "errors" : [ { + # "status" : 404, + # "message" : "Not Found" + # } ] + # } + for error in err.get('errors', []): + err_status = error.get('status', -1) + err_msg = error.get('message', '') + if err_status == 401: + msg = 'Authentication error[{}]{}'.format(err_status, + (': {}'.format( + err_msg)) if err_msg else '') + elif err_status == 404: + msg = last_error + else: + msg = 'Error[{}]{}'.format(err_status, + (': {}'.format(err_msg)) if err_msg else '') + if last_error != msg: + self._log.error(msg) + last_error = msg + return packagefound + + + def get_packages(self, source, parser, downloader, list_or_file_path, property_validate=True): """ @@ -240,6 +251,7 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v last_error = '' _pkg_name = _paths['params'][_pkg_name_column] # выбор нужного альтернативного пути +#_altpath - альтернатива _paths if downloader.altsearch: for item in altpaths: if item['params'][_pkg_name_column] == _pkg_name: @@ -255,12 +267,19 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v } ) ) - for _sub_paths in _paths['paths']: - self._tmp_params = dict(_paths['params']) - self._log.info('repo: {}'.format(_sub_paths['repo'])) - # обработка конкретного пути - self.searchpackage(downloader, _sub_paths, parser, _paths, _art_auth_etc, property_validate) + + if downloader.altsearch: + if self.searchpackage(downloader, parser, _altpath, _art_auth_etc, property_validate): + print('FOUND ALTERNATIVE PACKAGE: {}!'.format(_pkg_name)) + break + elif self.searchpackage(downloader, parser, _paths, _art_auth_etc, property_validate): + print ('FOUND PACKAGE{}!'.format(_pkg_name)) + break + else: + print('NOT FOUND!!') + else: + isfound = self.searchpackage(downloader, parser, _paths, _art_auth_etc, property_validate) _package = None @@ -391,7 +410,6 @@ def pkg_stat(package): def get_usedby(self, source, parser, downloader, list_or_file_path, property_validate=True): """ - :param source: :param parser: :param downloader: From 0a289e279f85690f636d2321be3b1012fdcb8350 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Tue, 1 Nov 2022 17:34:02 +0300 Subject: [PATCH 19/24] fix formatting --- crosspm/adapters/artifactoryaql.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index 5f08bd2..86c5550 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -76,7 +76,6 @@ def artifactory_search(self, _tmp_params, _art_auth_etc, _file_name_pattern, _pa return response - def searchpackage(self, downloader, parser, _paths, _art_auth_etc, property_validate): packagefound = False for _sub_paths in _paths['paths']: @@ -180,9 +179,6 @@ def searchpackage(self, downloader, parser, _paths, _art_auth_etc, property_vali last_error = msg return packagefound - - - def get_packages(self, source, parser, downloader, list_or_file_path, property_validate=True): """ :param source: @@ -233,7 +229,6 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v list_or_file_path_alt = self.get_alt_paths_list(list_or_file_path, downloader) altpaths = parser.get_paths(list_or_file_path_alt, source) - for _paths in parser.get_paths(list_or_file_path, source): _altpath = [] # If "parser"-column specified - find only in this parser @@ -248,7 +243,7 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v self._packages = [] self._params_found = {} self._params_found_raw = {} - last_error = '' + self.last_error = '' _pkg_name = _paths['params'][_pkg_name_column] # выбор нужного альтернативного пути #_altpath - альтернатива _paths @@ -274,12 +269,12 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v print('FOUND ALTERNATIVE PACKAGE: {}!'.format(_pkg_name)) break elif self.searchpackage(downloader, parser, _paths, _art_auth_etc, property_validate): - print ('FOUND PACKAGE{}!'.format(_pkg_name)) + print('FOUND PACKAGE{}!'.format(_pkg_name)) break else: print('NOT FOUND!!') else: - isfound = self.searchpackage(downloader, parser, _paths, _art_auth_etc, property_validate) + self.searchpackage(downloader, parser, _paths, _art_auth_etc, property_validate) _package = None @@ -449,7 +444,7 @@ def get_usedby(self, source, parser, downloader, list_or_file_path, property_val self._packages = [] self._params_found = {} self._params_found_raw = {} - last_error = '' + self.last_error = '' _pkg_name = _paths['params'][_pkg_name_col] if _pkg_name != _pkg_name_old: _pkg_name_old = _pkg_name @@ -539,13 +534,13 @@ def get_usedby(self, source, parser, downloader, list_or_file_path, property_val (': {}'.format( err_msg)) if err_msg else '') elif err_status == 404: - msg = last_error + msg = self.last_error else: msg = 'Error[{}]{}'.format(err_status, (': {}'.format(err_msg)) if err_msg else '') - if last_error != msg: + if self.last_error != msg: self._log.error(msg) - last_error = msg + self.last_error = msg return _packages_found From e45fc72affa1d8a2ed4c5ab52a58895a33ac3d20 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Wed, 2 Nov 2022 19:53:33 +0300 Subject: [PATCH 20/24] seems like it works --- crosspm/adapters/artifactoryaql.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index 86c5550..8a0d5c4 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -44,9 +44,9 @@ def get_alt_paths_list(self, list_or_file_path, downloader): list_or_file_path_alt['raw'].remove(element) return list_or_file_path_alt - def artifactory_search(self, _tmp_params, _art_auth_etc, _file_name_pattern, _path_pattern, _path_fixed): - _artifactory_server = _tmp_params['server'] - _search_repo = _tmp_params['repo'] + def artifactory_search(self, _art_auth_etc, _file_name_pattern, _path_pattern, _path_fixed): + _artifactory_server = self._tmp_params['server'] + _search_repo = self._tmp_params['repo'] # Get AQL path pattern, with fixed part path, without artifactory url and repository name _aql_path_pattern = _path_fixed[len(_artifactory_server) + 1 + len(_search_repo) + 1:] @@ -110,7 +110,7 @@ def searchpackage(self, downloader, parser, _paths, _art_auth_etc, property_vali # непосредственно, поиск в артифактории try: - searchresults = self.artifactory_search(self._tmp_params, _art_auth_etc, + searchresults = self.artifactory_search(_art_auth_etc, _file_name_pattern, _path_pattern, _path_fixed) _found_paths = searchresults.json() @@ -170,13 +170,13 @@ def searchpackage(self, downloader, parser, _paths, _art_auth_etc, property_vali (': {}'.format( err_msg)) if err_msg else '') elif err_status == 404: - msg = last_error + msg = self.last_error else: msg = 'Error[{}]{}'.format(err_status, (': {}'.format(err_msg)) if err_msg else '') - if last_error != msg: + if self.last_error != msg: self._log.error(msg) - last_error = msg + self.last_error = msg return packagefound def get_packages(self, source, parser, downloader, list_or_file_path, property_validate=True): @@ -246,7 +246,7 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v self.last_error = '' _pkg_name = _paths['params'][_pkg_name_column] # выбор нужного альтернативного пути -#_altpath - альтернатива _paths +# _altpath - альтернатива _paths if downloader.altsearch: for item in altpaths: if item['params'][_pkg_name_column] == _pkg_name: @@ -264,18 +264,15 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v ) # обработка конкретного пути - if downloader.altsearch: + if downloader.altsearch and _altpath != [] and _altpath['params'][_pkg_name_column] == _pkg_name: if self.searchpackage(downloader, parser, _altpath, _art_auth_etc, property_validate): print('FOUND ALTERNATIVE PACKAGE: {}!'.format(_pkg_name)) - break elif self.searchpackage(downloader, parser, _paths, _art_auth_etc, property_validate): print('FOUND PACKAGE{}!'.format(_pkg_name)) - break else: print('NOT FOUND!!') else: self.searchpackage(downloader, parser, _paths, _art_auth_etc, property_validate) - _package = None # HACK for prefer-local From 89df3d2e58557e208ed9fb928b0a0d8d5949ed71 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Wed, 2 Nov 2022 19:57:35 +0300 Subject: [PATCH 21/24] fix unused variable self._packed_cache_p --- crosspm/adapters/artifactoryaql.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index 8a0d5c4..55763a9 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -101,7 +101,7 @@ def searchpackage(self, downloader, parser, _paths, _art_auth_etc, property_vali self._packed_exist = os.path.isfile(_path_packed) if self._packed_exist: self._log.info("Skip searching, use package cache in path {}".format(_path_packed)) - _packed_cache_params = param + self._packed_cache_params = param break # break check local cache if self._packed_exist: break # break connect to artifactory @@ -222,7 +222,7 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v _packages_found = OrderedDict() _pkg_name_old = "" self._packed_exist = False - _packed_cache_params = None + self._packed_cache_params = None self._log.info('parser: {}'.format(parser._name)) if downloader.altsearch: @@ -278,12 +278,12 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v # HACK for prefer-local if self._packed_exist: # HACK - Normalize params for cached archive - for key, value in _packed_cache_params.items(): + for key, value in self._packed_cache_params.items(): if isinstance(value, list): value = ['' if x is None else x for x in value] - _packed_cache_params[key] = value + self._packed_cache_params[key] = value _package = Package(_pkg_name, None, _paths['params'], downloader, self, parser, - _packed_cache_params, list_or_file_path['raw'], {}, in_cache=True) + self._packed_cache_params, list_or_file_path['raw'], {}, in_cache=True) # END HACK if self._packages: _tmp = copy.deepcopy(self._params_found) From 8ed577c540314b86b7cd83a884257b8470e30886 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Mon, 7 Nov 2022 18:15:15 +0300 Subject: [PATCH 22/24] fix comments and messages --- crosspm/adapters/artifactoryaql.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index 55763a9..8cb1ff4 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -245,8 +245,7 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v self._params_found_raw = {} self.last_error = '' _pkg_name = _paths['params'][_pkg_name_column] -# выбор нужного альтернативного пути -# _altpath - альтернатива _paths +# _altpath is alternative to _paths if downloader.altsearch: for item in altpaths: if item['params'][_pkg_name_column] == _pkg_name: @@ -262,15 +261,14 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v } ) ) -# обработка конкретного пути if downloader.altsearch and _altpath != [] and _altpath['params'][_pkg_name_column] == _pkg_name: if self.searchpackage(downloader, parser, _altpath, _art_auth_etc, property_validate): print('FOUND ALTERNATIVE PACKAGE: {}!'.format(_pkg_name)) elif self.searchpackage(downloader, parser, _paths, _art_auth_etc, property_validate): - print('FOUND PACKAGE{}!'.format(_pkg_name)) + print('FOUND PACKAGE INSTEAD OF ALTERNATIVE: {}!'.format(_pkg_name)) else: - print('NOT FOUND!!') + print('{} NOT FOUND!!'.format(_pkg_name)) else: self.searchpackage(downloader, parser, _paths, _art_auth_etc, property_validate) _package = None From ff3ed469ab44e49d80834290acde42a6891cba3a Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Tue, 8 Nov 2022 14:21:39 +0300 Subject: [PATCH 23/24] fix docopt text for altsearch --- crosspm/cpm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crosspm/cpm.py b/crosspm/cpm.py index 5dd347d..0936a74 100644 --- a/crosspm/cpm.py +++ b/crosspm/cpm.py @@ -34,7 +34,7 @@ --recursive=VALUE Process all packages recursively to find and lock all dependencies --prefer-local Do not search package if exist in cache --stdout Print info and debug message to STDOUT, error to STDERR. Otherwise - all messages to STDERR - --altsearch=VALUE First search artifact in current branch repo then if not found in branch from config + --altsearch=VALUE First search artifact in VALUE branch then if it's not found in branch from config """ # noqa From c55345b28ea5b1572cffc4eb9bf7d39d3a8253e0 Mon Sep 17 00:00:00 2001 From: Alexey Solovyev Date: Tue, 8 Nov 2022 16:02:30 +0300 Subject: [PATCH 24/24] change altsearch parameter to --altsearch-branch --- crosspm/cpm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crosspm/cpm.py b/crosspm/cpm.py index 0936a74..edc9cfb 100644 --- a/crosspm/cpm.py +++ b/crosspm/cpm.py @@ -34,7 +34,7 @@ --recursive=VALUE Process all packages recursively to find and lock all dependencies --prefer-local Do not search package if exist in cache --stdout Print info and debug message to STDOUT, error to STDERR. Otherwise - all messages to STDERR - --altsearch=VALUE First search artifact in VALUE branch then if it's not found in branch from config + --altsearch-branch=VALUE First search artifact in VALUE branch then if it's not found in branch from config """ # noqa @@ -128,9 +128,9 @@ def __init__(self, args=None, throw_exceptions=None, return_result=False): else: raise Exception("Unknown value to --recursive: {}".format(recursive_str)) - if self._args['--altsearch']: + if self._args['--altsearch-branch']: self.altsearch = True - self.altsearch_branch = self._args['--altsearch'] + self.altsearch_branch = self._args['--altsearch-branch'] self._log.info("Alternative search is on. Priority searching in [%s] branch", self.altsearch_branch) if isinstance(self._args, str): @@ -345,7 +345,7 @@ def command(self, command_): 'out_format': ['--out-format', ''], 'output': ['--output', ''], 'output_template': ['--output-template', ''], - 'altsearch': ['--altsearch', ''], + 'altsearch': ['--altsearch-branch', ''], # 'out_prefix': ['--out-prefix', ''], # 'depslock_path': ['--depslock-path', ''], }