From 387c6ac3adc8db6702a57ffee755d703a2650180 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Sat, 2 Apr 2022 07:04:57 +0300 Subject: [PATCH 1/4] feat(structure_handler): initialize conditional steps logic (#710) --- tests/test_conditional_steps.py | 66 ++++++++++++++++++++++++++ universum/configuration_support.py | 8 ++++ universum/modules/launcher.py | 5 +- universum/modules/structure_handler.py | 31 ++++++++++-- 4 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 tests/test_conditional_steps.py diff --git a/tests/test_conditional_steps.py b/tests/test_conditional_steps.py new file mode 100644 index 00000000..ca1fa92c --- /dev/null +++ b/tests/test_conditional_steps.py @@ -0,0 +1,66 @@ +import re +import inspect +import pytest + +from universum import __main__ + + +true_branch_step_name = "true_branch" +false_branch_step_name = "false_branch" + + +def test_conditional_true_branch(tmpdir, capsys): + check_conditional_step_success(tmpdir, capsys, conditional_step_passed=True) + + +def test_conditional_false_branch(tmpdir, capsys): + check_conditional_step_success(tmpdir, capsys, conditional_step_passed=False) + + +def check_conditional_step_success(tmpdir, capsys, conditional_step_passed): + config_file = build_config_file(tmpdir, conditional_step_passed) + check_conditional_step(tmpdir, capsys, config_file, conditional_step_passed) + + +def build_config_file(tmpdir, conditional_step_passed): + conditional_step_exit_code = 0 if conditional_step_passed else 1 + + config = inspect.cleandoc(f''' + from universum.configuration_support import Configuration, Step + + true_branch_step = Step(name='{true_branch_step_name}', command=['touch', '{true_branch_step_name}']) + false_branch_step = Step(name='{false_branch_step_name}', command=['touch', '{false_branch_step_name}']) + conditional_step = Configuration([dict(name='conditional', + command=['bash', '-c', 'exit {conditional_step_exit_code}'], + if_succeeded=true_branch_step, if_failed=false_branch_step)]) + + configs = conditional_step + ''') + + config_file = tmpdir.join("configs.py") + config_file.write_text(config, "utf-8") + + return config_file + + +def check_conditional_step(tmpdir, capsys, config_file, conditional_step_passed): + artifacts_dir = tmpdir.join("artifacts") + params = ["-vt", "none", + "-fsd", str(tmpdir), + "-ad", str(artifacts_dir), + "--clean-build", + "-o", "console"] + params.extend(["-cfg", str(config_file)]) + + return_code = __main__.main(params) + assert return_code == 0 + + captured = capsys.readouterr() + print(captured.out) + conditional_succeeded_regexp = r"\] conditional.*Success.*\| 5\.2" + assert re.search(conditional_succeeded_regexp, captured.out, re.DOTALL) + + expected_log = true_branch_step_name if conditional_step_passed else false_branch_step_name + unexpected_log = false_branch_step_name if conditional_step_passed else true_branch_step_name + assert expected_log in captured.out + assert not unexpected_log in captured diff --git a/universum/configuration_support.py b/universum/configuration_support.py index 3bdb8d11..69752ceb 100644 --- a/universum/configuration_support.py +++ b/universum/configuration_support.py @@ -170,6 +170,8 @@ def __init__(self, pass_tag: str = '', fail_tag: str = '', if_env_set: str = '', + if_succeeded = None, + if_failed = None, **kwargs) -> None: self.name: str = name self.directory: str = directory @@ -185,6 +187,9 @@ def __init__(self, self.pass_tag: str = pass_tag self.fail_tag: str = fail_tag self.if_env_set: str = if_env_set + self.if_succeeded = if_succeeded + self.if_failed = if_failed + self.is_conditional = self.if_succeeded or self.if_failed self.children: Optional['Configuration'] = None self._extras: Dict[str, str] = {} for key, value in kwargs.items(): @@ -391,6 +396,9 @@ def __add__(self, other: 'Step') -> 'Step': pass_tag=self.pass_tag + other.pass_tag, fail_tag=self.fail_tag + other.fail_tag, if_env_set=self.if_env_set + other.if_env_set, + # FIXME: This is a dummy implementation. Define addition logic and implement it. + if_succeeded=other.if_succeeded, + if_failed=other.if_failed, **combine(self._extras, other._extras) ) diff --git a/universum/modules/launcher.py b/universum/modules/launcher.py index e3e2bb6a..6be8a363 100644 --- a/universum/modules/launcher.py +++ b/universum/modules/launcher.py @@ -282,8 +282,9 @@ def finalize(self) -> None: text = utils.trim_and_convert_to_unicode(text) if self.file: self.file.write(text + "\n") - self.add_tag(self.configuration.fail_tag) - self._error = text + if not self.configuration.is_conditional: + self.add_tag(self.configuration.fail_tag) + self._error = text return self.add_tag(self.configuration.pass_tag) diff --git a/universum/modules/structure_handler.py b/universum/modules/structure_handler.py index f4758beb..d19ba328 100644 --- a/universum/modules/structure_handler.py +++ b/universum/modules/structure_handler.py @@ -148,10 +148,10 @@ def block(self, *, block_name: str, pass_errors: bool) -> Generator: yield except SilentAbortException: raise - except CriticalCiException as e: + except CriticalCiException as e: # system/environment step failed self.fail_current_block(str(e)) raise SilentAbortException() from e - except Exception as e: + except Exception as e: # unexpected failure, should not occur if pass_errors is True: raise self.fail_current_block(str(e)) @@ -244,12 +244,13 @@ def execute_steps_recursively(self, parent: Step, with self.block(block_name=step_label, pass_errors=True): current_step_failed = not self.execute_steps_recursively(merged_item, child.children, step_executor, skip_execution) + elif child.is_conditional: + current_step_failed = not self.execute_conditional_step(merged_item, step_executor) else: if merged_item.finish_background and self.active_background_steps: self.out.log("All ongoing background steps should be finished before next step execution") if not self.report_background_steps(): skip_execution = True - current_step_failed = not self.process_one_step(merged_item, step_executor, skip_execution) if current_step_failed: @@ -264,6 +265,20 @@ def execute_steps_recursively(self, parent: Step, return not some_step_failed + + def execute_conditional_step(self, step, step_executor): + self.configs_current_number += 1 + step_name = self._build_step_name(step.name) + conditional_step_succeeded = False + with self.block(block_name=step_name, pass_errors=True): + process = self.execute_one_step(step, step_executor); + conditional_step_succeeded = not process.get_error() + step_to_execute = step.if_succeeded if conditional_step_succeeded else step.if_failed + return self.execute_steps_recursively(parent=Step(), + children=Configuration([step_to_execute]), + step_executor=step_executor, + skip_execution=False) + def report_background_steps(self) -> bool: result: bool = True for item in self.active_background_steps: @@ -282,7 +297,10 @@ def report_background_steps(self) -> bool: return result def execute_step_structure(self, configs: Configuration, step_executor) -> None: - self.configs_total_count = sum(1 for _ in configs.all()) + for config in configs.all(): + self.configs_total_count += 1 + if config.if_succeeded or config.if_failed: + self.configs_total_count += 1 self.step_num_len = len(str(self.configs_total_count)) self.group_numbering = f" [ {'':{self.step_num_len}}+{'':{self.step_num_len}} ] " @@ -292,6 +310,11 @@ def execute_step_structure(self, configs: Configuration, step_executor) -> None: with self.block(block_name="Reporting background steps", pass_errors=False): self.report_background_steps() + def _build_step_name(self, name): + step_num_len = len(str(self.configs_total_count)) + numbering = f" [ {self.configs_current_number:>{step_num_len}}/{self.configs_total_count} ] " + return numbering + name + class HasStructure(Module): structure_factory: ClassVar = Dependency(StructureHandler) From 1d21884a6c9b1897725a95d9019887dc5c9af981 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Fri, 7 Oct 2022 12:01:38 +0300 Subject: [PATCH 2/4] simplify 'is_conditional' check --- universum/modules/structure_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/universum/modules/structure_handler.py b/universum/modules/structure_handler.py index d19ba328..1587df9d 100644 --- a/universum/modules/structure_handler.py +++ b/universum/modules/structure_handler.py @@ -299,7 +299,7 @@ def report_background_steps(self) -> bool: def execute_step_structure(self, configs: Configuration, step_executor) -> None: for config in configs.all(): self.configs_total_count += 1 - if config.if_succeeded or config.if_failed: + if config.is_conditional: self.configs_total_count += 1 self.step_num_len = len(str(self.configs_total_count)) self.group_numbering = f" [ {'':{self.step_num_len}}+{'':{self.step_num_len}} ] " From 6a04c6034925d9f165f68d837747d1d56b00a70e Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Mon, 10 Oct 2022 07:47:01 +0300 Subject: [PATCH 3/4] remove semicolon --- universum/modules/structure_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/universum/modules/structure_handler.py b/universum/modules/structure_handler.py index 1587df9d..783d22d0 100644 --- a/universum/modules/structure_handler.py +++ b/universum/modules/structure_handler.py @@ -271,7 +271,7 @@ def execute_conditional_step(self, step, step_executor): step_name = self._build_step_name(step.name) conditional_step_succeeded = False with self.block(block_name=step_name, pass_errors=True): - process = self.execute_one_step(step, step_executor); + process = self.execute_one_step(step, step_executor) conditional_step_succeeded = not process.get_error() step_to_execute = step.if_succeeded if conditional_step_succeeded else step.if_failed return self.execute_steps_recursively(parent=Step(), From 6a0fe06beca7af42fb8701ba366d2a099808af8f Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Wed, 19 Oct 2022 08:02:28 +0300 Subject: [PATCH 4/4] refactor TeamCity tag adding --- universum/configuration_support.py | 4 ++-- universum/modules/launcher.py | 34 +++++++++++++++--------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/universum/configuration_support.py b/universum/configuration_support.py index 69752ceb..9c5e4ac6 100644 --- a/universum/configuration_support.py +++ b/universum/configuration_support.py @@ -115,9 +115,9 @@ class Step: A tag used to mark successful TeamCity builds. This tag can be set independenty of `fail_tag` value per each step. The value should be set to a strings without spaces as acceptable by TeamCity as tags. Every tag is added (if matching condition) after executing build step it is set in, - not in the end of all run. + not in the end of all run. Not applicable for conditional steps. fail_tag - A tag used to mark failed TemCity builds. See `pass_tag` for details. + A tag used to mark failed TemCity builds. See `pass_tag` for details. Not applicable for conditional steps. Each parameter is optional, and is substituted with a falsy value, if omitted. diff --git a/universum/modules/launcher.py b/universum/modules/launcher.py index 6be8a363..fe0b974e 100644 --- a/universum/modules/launcher.py +++ b/universum/modules/launcher.py @@ -248,16 +248,6 @@ def handle_stderr(self, line: str) -> None: else: self.out.log_stderr(line) - def add_tag(self, tag: str) -> None: - if not tag: - return - - request: Response = self.send_tag(tag) - if request.status_code != 200: - self.out.log_error(request.text) - else: - self.out.log("Tag '" + tag + "' added to build.") - def finalize(self) -> None: self._error = None if not self._needs_finalization: @@ -282,15 +272,12 @@ def finalize(self) -> None: text = utils.trim_and_convert_to_unicode(text) if self.file: self.file.write(text + "\n") - if not self.configuration.is_conditional: - self.add_tag(self.configuration.fail_tag) - self._error = text - return - - self.add_tag(self.configuration.pass_tag) - return + self._error = text finally: + tag: Optional[str] = self._get_teamcity_build_tag() + if tag: + self._assign_teamcity_build_tag(tag) self.handle_stdout() if self.file: self.file.close() @@ -308,6 +295,19 @@ def _handle_postponed_out(self) -> None: item[0](item[1]) self._postponed_out = [] + def _get_teamcity_build_tag(self) -> Optional[str]: + if self.configuration.is_conditional: + return None # conditional steps always succeed, no sense to set a tag + tag: str = self.configuration.fail_tag if self._error else self.configuration.pass_tag + return tag # can be also None if not set for current Configuration + + def _assign_teamcity_build_tag(self, tag: str) -> None: + response: Response = self.send_tag(tag) + if response.status_code != 200: + self.out.log_error(response.text) + else: + self.out.log("Tag '" + tag + "' added to build.") + class Launcher(ProjectDirectory, HasOutput, HasStructure, HasErrorState): artifacts_factory = Dependency(artifact_collector.ArtifactCollector)