From 2cb6e63c02caa174f650b1fe7b7a1ae00e61bea3 Mon Sep 17 00:00:00 2001 From: Jonathan Drake Date: Mon, 29 Mar 2021 17:04:03 -0700 Subject: [PATCH 1/3] imports --- .flake8 | 2 +- .pre-commit-config.yaml | 1 + fluxio_parser/__init__.py | 4 ++-- fluxio_parser/parser.py | 4 ++-- fluxio_parser/resource_decorators.py | 4 ++-- fluxio_parser/states/__init__.py | 20 +++++++++------- fluxio_parser/states/base.py | 9 ++++--- fluxio_parser/states/choice.py | 18 +++++++++----- fluxio_parser/states/fail.py | 2 +- fluxio_parser/states/map_.py | 24 ++++++++++++++----- fluxio_parser/states/parallel.py | 4 ++-- fluxio_parser/states/pass_.py | 13 ++++++---- fluxio_parser/states/succeed.py | 9 +++---- fluxio_parser/states/tasks/__init__.py | 17 ++++++------- fluxio_parser/states/tasks/base.py | 15 +++++++----- fluxio_parser/states/tasks/codebuild.py | 2 +- fluxio_parser/states/tasks/ecs.py | 2 +- fluxio_parser/states/tasks/ecs_worker.py | 2 +- fluxio_parser/states/tasks/lambda_function.py | 4 ++-- .../states/tasks/lambda_pexpm_runner.py | 4 ++-- fluxio_parser/states/tasks/retry.py | 6 ++++- fluxio_parser/states/tasks/state_machine.py | 2 +- fluxio_parser/states/wait.py | 6 ++--- fluxio_parser/transformers/__init__.py | 7 +++--- fluxio_parser/transformers/data_dict.py | 2 +- fluxio_parser/util.py | 2 +- fluxio_parser/visitors/__init__.py | 9 +++---- fluxio_parser/visitors/event_processor.py | 9 +++---- fluxio_parser/visitors/script.py | 15 +++++++----- fluxio_parser/visitors/state_machine.py | 6 ++--- fluxio_parser/visitors/task.py | 6 ++--- pyproject.toml | 2 +- tests/test_event_processor_visitor.py | 4 ++-- 33 files changed, 138 insertions(+), 98 deletions(-) diff --git a/.flake8 b/.flake8 index c95d0bf..84fdf3e 100644 --- a/.flake8 +++ b/.flake8 @@ -8,7 +8,7 @@ ignore = E203,E501,E731,W503,W605 import-order-style = google # Packages added in this list should be added to the setup.cfg file as well application-import-names = - pypants + fluxio_parser exclude = *vendor* .venv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ae3d511..db47e79 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,6 +24,7 @@ repos: args: [--markdown-linebreak-ext=md] # Need to define stages explicitly since `default_stages` was not being respected stages: [commit] + - id: end-of-file-fixer - repo: local hooks: diff --git a/fluxio_parser/__init__.py b/fluxio_parser/__init__.py index affbe55..8f787cf 100644 --- a/fluxio_parser/__init__.py +++ b/fluxio_parser/__init__.py @@ -1,5 +1,5 @@ """Main exports""" -from .parser import parse_project_tree # noqa: F401 +from fluxio_parser.parser import parse_project_tree # noqa: F401 -__version__ = "0.3.0" +__version__ = "0.3.1" diff --git a/fluxio_parser/parser.py b/fluxio_parser/parser.py index 98aab2c..f006314 100644 --- a/fluxio_parser/parser.py +++ b/fluxio_parser/parser.py @@ -2,8 +2,8 @@ import ast -from .transformers import ScriptTransformer -from .visitors import ScriptVisitor +from fluxio_parser.transformers import ScriptTransformer +from fluxio_parser.visitors import ScriptVisitor def parse_project_tree( diff --git a/fluxio_parser/resource_decorators.py b/fluxio_parser/resource_decorators.py index 1d40281..c8310fb 100644 --- a/fluxio_parser/resource_decorators.py +++ b/fluxio_parser/resource_decorators.py @@ -2,8 +2,8 @@ import ast from typing import Any, Optional -from .exceptions import assert_supported_operation -from .util import CallableOption, GET_VALUE_MAP +from fluxio_parser.exceptions import assert_supported_operation +from fluxio_parser.util import CallableOption, GET_VALUE_MAP def get_subscribe_status(node: Any, visitor: Optional[ast.NodeVisitor]) -> str: diff --git a/fluxio_parser/states/__init__.py b/fluxio_parser/states/__init__.py index b1ee74b..685bdb0 100644 --- a/fluxio_parser/states/__init__.py +++ b/fluxio_parser/states/__init__.py @@ -1,9 +1,11 @@ -from .base import State -from .choice import ChoiceState -from .fail import FailState -from .map_ import MapState -from .parallel import ParallelState -from .pass_ import PassState -from .succeed import SucceedState -from .tasks import create_task_state, TaskState -from .wait import WaitState +"""Contains exports for the states subpackage""" +from fluxio_parser.states.base import State +from fluxio_parser.states.choice import ChoiceState +from fluxio_parser.states.fail import FailState +from fluxio_parser.states.map_ import MapState +from fluxio_parser.states.parallel import ParallelState +from fluxio_parser.states.pass_ import PassState +from fluxio_parser.states.succeed import SucceedState +from fluxio_parser.states.tasks import create_task_state +from fluxio_parser.states.tasks.base import TaskState +from fluxio_parser.states.wait import WaitState diff --git a/fluxio_parser/states/base.py b/fluxio_parser/states/base.py index 4e773db..b6d5926 100644 --- a/fluxio_parser/states/base.py +++ b/fluxio_parser/states/base.py @@ -4,7 +4,7 @@ import astor import networkx as nx -from ..util import hash_node +from fluxio_parser.util import hash_node class StateMachineFragment: @@ -24,10 +24,12 @@ def __init__(self, state_graph: nx.DiGraph, key: str, ast_node: Any) -> None: self._source = astor.to_source(ast_node).strip() self._hash = hash_node(ast_node) - def __str__(self): + def __str__(self) -> str: + """Returns string representation of the object""" return str(self.key) - def __repr__(self): + def __repr__(self) -> str: + """Returns REPR representation of the object""" return f"{self.__class__.__name__}(key={self.key})" def _set_end_or_next(self, data: Dict) -> Dict: @@ -40,6 +42,7 @@ def _set_end_or_next(self, data: Dict) -> Dict: Returns: modified input data + """ edges = self.edges if len(edges) == 0: diff --git a/fluxio_parser/states/choice.py b/fluxio_parser/states/choice.py index c1b496e..9658e33 100644 --- a/fluxio_parser/states/choice.py +++ b/fluxio_parser/states/choice.py @@ -1,10 +1,13 @@ """Contains the classes that represent the AWS Step Functions Choice State""" import ast -from typing import Any, Dict +from typing import Any, Dict, TYPE_CHECKING -from ..exceptions import assert_supported_operation, UnsupportedOperation -from ..util import convert_input_data_ref, hash_node -from .base import State, StateMachineFragment +from fluxio_parser.exceptions import assert_supported_operation, UnsupportedOperation +from fluxio_parser.states.base import State, StateMachineFragment +from fluxio_parser.util import convert_input_data_ref, hash_node + +if TYPE_CHECKING: + import networkx as nx # Map of either: # * tuple of: @@ -219,13 +222,15 @@ class ChoiceState(State): } """ - def __init__(self, state_graph: "nx.DiGraph", key: str, ast_node: Any) -> None: - """ + def __init__(self, state_graph: nx.DiGraph, key: str, ast_node: Any) -> None: + """Initializer + Args: state_graph: DAG of state machine fragments with edges between them key: Key of the fragment in the state machine's States value. This only really applies to States, not generic fragments. ast_node: AST node for this fragment in the .sfn file + """ super().__init__(state_graph, key, ast_node) self.choice_branches = [] @@ -242,6 +247,7 @@ def add_choice_branch(self, node: Any) -> ChoiceBranch: Returns: new ChoiceBranch instance + """ choice_branch = ChoiceBranch( self.state_graph, f"ChoiceBranch-{hash_node(node)}", node diff --git a/fluxio_parser/states/fail.py b/fluxio_parser/states/fail.py index 538cd8a..2b90e13 100644 --- a/fluxio_parser/states/fail.py +++ b/fluxio_parser/states/fail.py @@ -1,7 +1,7 @@ """Contains the class that represents the AWS Step Functions Fail State""" from typing import Dict -from .base import State +from fluxio_parser.states.base import State class FailState(State): diff --git a/fluxio_parser/states/map_.py b/fluxio_parser/states/map_.py index ea40202..e5bac2a 100644 --- a/fluxio_parser/states/map_.py +++ b/fluxio_parser/states/map_.py @@ -1,9 +1,17 @@ """Contains the classes that represent the AWS Step Functions Map State""" import ast -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, TYPE_CHECKING -from ..util import CallableOption, convert_input_data_ref, GET_VALUE_MAP, parse_options -from .base import State +from fluxio_parser.states.base import State +from fluxio_parser.util import ( + CallableOption, + convert_input_data_ref, + GET_VALUE_MAP, + parse_options, +) + +if TYPE_CHECKING: + import networkx as nx # Map of task option name to an option schema OPTION_MAP = { @@ -48,18 +56,20 @@ def iterator_function(data): def __init__( self, - state_graph: "nx.DiGraph", + state_graph: nx.DiGraph, key: str, ast_node: Any, - iterator: "StateMachineVisitor", + iterator: "StateMachineVisitor", # noqa: F821 ) -> None: - """ + """Initializer + Args: state_graph: DAG of state machine fragments with edges between them key: Key of the fragment in the state machine's States value. This only really applies to States, not generic fragments. ast_node: AST node for this fragment in the .sfn file iterator: State machine visitor for the iterator function + """ super().__init__(state_graph, key, ast_node) self.iterator = iterator @@ -77,6 +87,7 @@ def _get_input_path(self) -> str: Returns: input path string + """ if len(self.ast_node.value.args) > 0: return convert_input_data_ref(self.ast_node.value.args[0]) @@ -91,6 +102,7 @@ def _get_result_path(self) -> Optional[str]: Returns: result path string or None + """ if isinstance(self.ast_node, ast.Assign) and len(self.ast_node.targets) > 0: return convert_input_data_ref(self.ast_node.targets[0]) diff --git a/fluxio_parser/states/parallel.py b/fluxio_parser/states/parallel.py index 122452f..e0411c8 100644 --- a/fluxio_parser/states/parallel.py +++ b/fluxio_parser/states/parallel.py @@ -3,10 +3,10 @@ import networkx as nx -from .base import State +from fluxio_parser.states.base import State if TYPE_CHECKING: - from ..visitors import StateMachineVisitor + from fluxio_parser.visitors import StateMachineVisitor class ParallelState(State): diff --git a/fluxio_parser/states/pass_.py b/fluxio_parser/states/pass_.py index 99377b8..b8d26d5 100644 --- a/fluxio_parser/states/pass_.py +++ b/fluxio_parser/states/pass_.py @@ -6,10 +6,13 @@ import astor -from ..constants import INVALID_RESULT_PATH_PATTERN, RESERVED_INPUT_DATA_KEYS -from ..exceptions import assert_supported_operation, UnsupportedOperation -from ..util import convert_input_data_ref -from .base import State +from fluxio_parser.constants import ( + INVALID_RESULT_PATH_PATTERN, + RESERVED_INPUT_DATA_KEYS, +) +from fluxio_parser.exceptions import assert_supported_operation, UnsupportedOperation +from fluxio_parser.states.base import State +from fluxio_parser.util import convert_input_data_ref class PassState(State): @@ -67,6 +70,7 @@ def _parse_result(self) -> Any: Returns: result value + """ if isinstance(self.ast_node, ast.Assign): source = astor.to_source(self.ast_node.value).strip() @@ -94,6 +98,7 @@ def _parse_result_path(self) -> str: Returns: result path string + """ if isinstance(self.ast_node, ast.Assign): result_path = convert_input_data_ref(self.ast_node.targets[0]) diff --git a/fluxio_parser/states/succeed.py b/fluxio_parser/states/succeed.py index 167003a..e3989a0 100644 --- a/fluxio_parser/states/succeed.py +++ b/fluxio_parser/states/succeed.py @@ -1,7 +1,7 @@ """Contains the class that represents the AWS Step Functions Succeed State""" from typing import Dict -from .base import State +from fluxio_parser.states.base import State class SucceedState(State): @@ -12,15 +12,12 @@ class SucceedState(State): See: https://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-succeed-state.html - For example:: - - return - - resolves to:: + For example, ``return`` resolves to:: { "Type": "Succeed" } + """ TERMINAL = True diff --git a/fluxio_parser/states/tasks/__init__.py b/fluxio_parser/states/tasks/__init__.py index 0123e73..e81723a 100644 --- a/fluxio_parser/states/tasks/__init__.py +++ b/fluxio_parser/states/tasks/__init__.py @@ -1,14 +1,14 @@ """Contains a factory function for creating new Task state instances""" from typing import Any, Union -from ...util import parse_options -from .base import OPTION_MAP, TaskState -from .codebuild import CodeBuildTaskState -from .ecs import ECSTaskState -from .ecs_worker import ECSWorkerTaskState -from .lambda_function import LambdaTaskState -from .lambda_pexpm_runner import LambdaPEXPMRunnerTaskState -from .state_machine import StateMachineTaskState +from fluxio_parser.states.tasks.base import OPTION_MAP +from fluxio_parser.states.tasks.codebuild import CodeBuildTaskState +from fluxio_parser.states.tasks.ecs import ECSTaskState +from fluxio_parser.states.tasks.ecs_worker import ECSWorkerTaskState +from fluxio_parser.states.tasks.lambda_function import LambdaTaskState +from fluxio_parser.states.tasks.lambda_pexpm_runner import LambdaPEXPMRunnerTaskState +from fluxio_parser.states.tasks.state_machine import StateMachineTaskState +from fluxio_parser.util import parse_options #: Map of service key to task state class TASK_STATE_MAP = { @@ -37,6 +37,7 @@ def create_task_state( Returns: new task state + """ service = visitor.attributes.get("service", "lambda") options = {} diff --git a/fluxio_parser/states/tasks/base.py b/fluxio_parser/states/tasks/base.py index 0a29adb..ccbeaa3 100644 --- a/fluxio_parser/states/tasks/base.py +++ b/fluxio_parser/states/tasks/base.py @@ -6,10 +6,15 @@ import astor -from ...constants import INVALID_RESULT_PATH_PATTERN, RESERVED_INPUT_DATA_KEYS -from ...exceptions import assert_supported_operation -from ...transformers import DataDictTransformer -from ...util import ( +from fluxio_parser.constants import ( + INVALID_RESULT_PATH_PATTERN, + RESERVED_INPUT_DATA_KEYS, +) +from fluxio_parser.exceptions import assert_supported_operation +from fluxio_parser.states.base import State, StateMachineFragment +from fluxio_parser.states.tasks.retry import Retry, RETRY_OPTION_MAP +from fluxio_parser.transformers import DataDictTransformer +from fluxio_parser.util import ( CallableOption, convert_input_data_ref, GET_VALUE_MAP, @@ -17,8 +22,6 @@ parse_options, serialize_error_name, ) -from ..base import State, StateMachineFragment -from .retry import Retry, RETRY_OPTION_MAP def convert_data_dict(node: Any, visitor: ast.NodeVisitor) -> str: diff --git a/fluxio_parser/states/tasks/codebuild.py b/fluxio_parser/states/tasks/codebuild.py index 4f12962..16e95a2 100644 --- a/fluxio_parser/states/tasks/codebuild.py +++ b/fluxio_parser/states/tasks/codebuild.py @@ -2,7 +2,7 @@ import json from typing import Dict, Set -from .base import TaskState +from fluxio_parser.states.tasks.base import TaskState class CodeBuildTaskState(TaskState): diff --git a/fluxio_parser/states/tasks/ecs.py b/fluxio_parser/states/tasks/ecs.py index 5b4572d..6a45b22 100644 --- a/fluxio_parser/states/tasks/ecs.py +++ b/fluxio_parser/states/tasks/ecs.py @@ -1,7 +1,7 @@ """Contains class for a Task State that runs an ECS task""" from typing import Dict, Set -from .base import TaskState +from fluxio_parser.states.tasks.base import TaskState class ECSTaskState(TaskState): diff --git a/fluxio_parser/states/tasks/ecs_worker.py b/fluxio_parser/states/tasks/ecs_worker.py index 9e01803..cf88404 100644 --- a/fluxio_parser/states/tasks/ecs_worker.py +++ b/fluxio_parser/states/tasks/ecs_worker.py @@ -1,7 +1,7 @@ """Contains class for a Task State that feeds an ECS worker""" from typing import Dict, Set -from .base import TaskState +from fluxio_parser.states.tasks.base import TaskState class ECSWorkerTaskState(TaskState): diff --git a/fluxio_parser/states/tasks/lambda_function.py b/fluxio_parser/states/tasks/lambda_function.py index ff491e3..1286f31 100644 --- a/fluxio_parser/states/tasks/lambda_function.py +++ b/fluxio_parser/states/tasks/lambda_function.py @@ -1,8 +1,8 @@ """Contains class used to represent a Task State that integrates with Lambda""" from typing import Dict, Set -from .base import TaskState -from .retry import Retry +from fluxio_parser.states.tasks.base import TaskState +from fluxio_parser.states.tasks.retry import Retry class LambdaTaskState(TaskState): diff --git a/fluxio_parser/states/tasks/lambda_pexpm_runner.py b/fluxio_parser/states/tasks/lambda_pexpm_runner.py index 19a8ec8..cd836c4 100644 --- a/fluxio_parser/states/tasks/lambda_pexpm_runner.py +++ b/fluxio_parser/states/tasks/lambda_pexpm_runner.py @@ -1,8 +1,8 @@ """Contains a class representing a Task State for Lambda via the PEXPM Runner""" from typing import Dict, Set -from .base import TaskState -from .retry import Retry +from fluxio_parser.states.tasks.base import TaskState +from fluxio_parser.states.tasks.retry import Retry class LambdaPEXPMRunnerTaskState(TaskState): diff --git a/fluxio_parser/states/tasks/retry.py b/fluxio_parser/states/tasks/retry.py index 12f1098..117545d 100644 --- a/fluxio_parser/states/tasks/retry.py +++ b/fluxio_parser/states/tasks/retry.py @@ -2,7 +2,11 @@ import ast from typing import Dict, List -from ...util import CallableOption, GET_VALUE_MAP, serialize_error_name # noqa +from fluxio_parser.util import ( # noqa + CallableOption, + GET_VALUE_MAP, + serialize_error_name, +) RETRY_OPTION_MAP = { "on_exceptions": CallableOption( diff --git a/fluxio_parser/states/tasks/state_machine.py b/fluxio_parser/states/tasks/state_machine.py index c04d598..7c949ed 100644 --- a/fluxio_parser/states/tasks/state_machine.py +++ b/fluxio_parser/states/tasks/state_machine.py @@ -1,7 +1,7 @@ """Contains class used to represent a Task State that integrates with Step Functions""" from typing import Dict, Set -from .base import TaskState +from fluxio_parser.states.tasks.base import TaskState class StateMachineTaskState(TaskState): diff --git a/fluxio_parser/states/wait.py b/fluxio_parser/states/wait.py index f501c5e..3a1e714 100644 --- a/fluxio_parser/states/wait.py +++ b/fluxio_parser/states/wait.py @@ -2,9 +2,9 @@ import ast from typing import Dict -from ..exceptions import UnsupportedOperation -from ..util import convert_input_data_ref -from .base import State +from fluxio_parser.exceptions import UnsupportedOperation +from fluxio_parser.states.base import State +from fluxio_parser.util import convert_input_data_ref class WaitState(State): diff --git a/fluxio_parser/transformers/__init__.py b/fluxio_parser/transformers/__init__.py index 9451137..340e5bb 100644 --- a/fluxio_parser/transformers/__init__.py +++ b/fluxio_parser/transformers/__init__.py @@ -1,3 +1,4 @@ -from .data_dict import DataDictTransformer -from .run_method import RunMethodTransformer -from .script import ScriptTransformer +"""Contains exports from the transformers subpackage""" +from fluxio_parser.transformers.data_dict import DataDictTransformer +from fluxio_parser.transformers.run_method import RunMethodTransformer +from fluxio_parser.transformers.script import ScriptTransformer diff --git a/fluxio_parser/transformers/data_dict.py b/fluxio_parser/transformers/data_dict.py index fae6aab..3eaef34 100644 --- a/fluxio_parser/transformers/data_dict.py +++ b/fluxio_parser/transformers/data_dict.py @@ -2,7 +2,7 @@ import ast from typing import Any -from ..util import convert_input_data_ref +from fluxio_parser.util import convert_input_data_ref class DataDictTransformer(ast.NodeTransformer): diff --git a/fluxio_parser/util.py b/fluxio_parser/util.py index fe2236d..3661d11 100644 --- a/fluxio_parser/util.py +++ b/fluxio_parser/util.py @@ -7,7 +7,7 @@ import astor -from .exceptions import assert_supported_operation, UnsupportedOperation +from fluxio_parser.exceptions import assert_supported_operation, UnsupportedOperation logger = logging.getLogger(__name__) diff --git a/fluxio_parser/visitors/__init__.py b/fluxio_parser/visitors/__init__.py index 1c12872..298d16e 100644 --- a/fluxio_parser/visitors/__init__.py +++ b/fluxio_parser/visitors/__init__.py @@ -1,4 +1,5 @@ -from .event_processor import EventProcessorVisitor -from .script import ScriptVisitor -from .state_machine import StateMachineVisitor -from .task import TaskVisitor +"""Contains exports for the visitors subpackage""" +from fluxio_parser.visitors.event_processor import EventProcessorVisitor +from fluxio_parser.visitors.script import ScriptVisitor +from fluxio_parser.visitors.state_machine import StateMachineVisitor +from fluxio_parser.visitors.task import TaskVisitor diff --git a/fluxio_parser/visitors/event_processor.py b/fluxio_parser/visitors/event_processor.py index 5e097a6..ae3a97e 100644 --- a/fluxio_parser/visitors/event_processor.py +++ b/fluxio_parser/visitors/event_processor.py @@ -3,7 +3,7 @@ import ast from typing import Any -from ..exceptions import assert_supported_operation, UnsupportedOperation +from fluxio_parser.exceptions import assert_supported_operation, UnsupportedOperation class EventProcessorVisitor(ast.NodeVisitor): @@ -43,9 +43,10 @@ def visit_AsyncFunctionDef(self, node: Any) -> None: ) arg_name_set = {arg.arg for arg in node.args.args} assert_supported_operation( - arg_name_set == {"message"}, - f"get_custom_tags_* methods must only accept a positional argument of message." - f" Provided: {', '.join(arg_name_set)}", + arg_name_set == {"message", "input_data", "state_data_client"}, + "get_custom_tags_* methods must only accept positional arguments of" + " (message, input_data, state_data_client)." + f" Provided: {', '.join([arg.arg for arg in node.args.args])}", node, ) diff --git a/fluxio_parser/visitors/script.py b/fluxio_parser/visitors/script.py index bb41d3d..2abb707 100644 --- a/fluxio_parser/visitors/script.py +++ b/fluxio_parser/visitors/script.py @@ -3,12 +3,15 @@ from collections import defaultdict from typing import Any, Dict, Set -from ..exceptions import assert_supported_operation, UnsupportedOperation # noqa -from ..resource_decorators import RESOURCE_DECORATOR_MAP # noqa -from ..util import parse_options # noqa -from .event_processor import EventProcessorVisitor # noqa -from .state_machine import StateMachineVisitor # noqa -from .task import TaskVisitor # noqa +from fluxio_parser.exceptions import ( # noqa + assert_supported_operation, + UnsupportedOperation, +) +from fluxio_parser.resource_decorators import RESOURCE_DECORATOR_MAP # noqa +from fluxio_parser.util import parse_options # noqa +from fluxio_parser.visitors.event_processor import EventProcessorVisitor # noqa +from fluxio_parser.visitors.state_machine import StateMachineVisitor # noqa +from fluxio_parser.visitors.task import TaskVisitor # noqa class ScriptVisitor(ast.NodeVisitor): diff --git a/fluxio_parser/visitors/state_machine.py b/fluxio_parser/visitors/state_machine.py index 9fbfbeb..8039cd8 100644 --- a/fluxio_parser/visitors/state_machine.py +++ b/fluxio_parser/visitors/state_machine.py @@ -7,8 +7,8 @@ import astor import networkx as nx -from ..exceptions import assert_supported_operation, UnsupportedOperation -from ..states import ( +from fluxio_parser.exceptions import assert_supported_operation, UnsupportedOperation +from fluxio_parser.states import ( ChoiceState, create_task_state, FailState, @@ -20,7 +20,7 @@ TaskState, WaitState, ) -from ..util import hash_node +from fluxio_parser.util import hash_node class StateMachineVisitor(ast.NodeVisitor): diff --git a/fluxio_parser/visitors/task.py b/fluxio_parser/visitors/task.py index 0d89b8e..5fe6fab 100644 --- a/fluxio_parser/visitors/task.py +++ b/fluxio_parser/visitors/task.py @@ -2,9 +2,9 @@ import ast from typing import Any, Callable, NamedTuple, Optional, Set, Union -from ..exceptions import assert_supported_operation, UnsupportedOperation -from ..transformers import RunMethodTransformer -from ..util import GET_VALUE_MAP +from fluxio_parser.exceptions import assert_supported_operation, UnsupportedOperation +from fluxio_parser.transformers import RunMethodTransformer +from fluxio_parser.util import GET_VALUE_MAP class Attribute(NamedTuple): diff --git a/pyproject.toml b/pyproject.toml index 7940a40..33a92d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "fluxio-parser" -version = "0.3.0" +version = "0.3.1" description = "Fluxio parser library" authors = ["Jonathan Drake "] license = "BSD-3-Clause" diff --git a/tests/test_event_processor_visitor.py b/tests/test_event_processor_visitor.py index d11aaa5..b893ef4 100644 --- a/tests/test_event_processor_visitor.py +++ b/tests/test_event_processor_visitor.py @@ -23,10 +23,10 @@ def test_ok(self): """Should not fail when proper methods set""" code = """ class MyEventProcessor(EventProcessor): - async def get_custom_tags_Foo(message): + async def get_custom_tags_Foo(message, input_data, state_data_client): pass - async def get_custom_tags_Bar(message): + async def get_custom_tags_Bar(message, input_data, state_data_client): pass """ tree = ast.parse(code) From 21ed8d1aab53461cc1acc9b563ee6b9854dba4ef Mon Sep 17 00:00:00 2001 From: Jonathan Drake Date: Mon, 29 Mar 2021 17:09:22 -0700 Subject: [PATCH 2/3] types --- fluxio_parser/states/tasks/retry.py | 6 +----- fluxio_parser/util.py | 4 ++-- fluxio_parser/visitors/script.py | 15 ++++++--------- fluxio_parser/visitors/task.py | 6 +++--- 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/fluxio_parser/states/tasks/retry.py b/fluxio_parser/states/tasks/retry.py index 117545d..5293e1b 100644 --- a/fluxio_parser/states/tasks/retry.py +++ b/fluxio_parser/states/tasks/retry.py @@ -2,11 +2,7 @@ import ast from typing import Dict, List -from fluxio_parser.util import ( # noqa - CallableOption, - GET_VALUE_MAP, - serialize_error_name, -) +from fluxio_parser.util import CallableOption, GET_VALUE_MAP, serialize_error_name RETRY_OPTION_MAP = { "on_exceptions": CallableOption( diff --git a/fluxio_parser/util.py b/fluxio_parser/util.py index 3661d11..9faf6e4 100644 --- a/fluxio_parser/util.py +++ b/fluxio_parser/util.py @@ -107,9 +107,9 @@ class CallableOption(NamedTuple): #: Getter function for extracting the value from the AST node get_value: Callable[[Any, ast.NodeVisitor], Any] #: Default value if the option is not provided - default_value: Any = None # noqa + default_value: Any = None #: Flag indicating that this option is required - required: bool = False # noqa + required: bool = False class OptionsMap(dict): diff --git a/fluxio_parser/visitors/script.py b/fluxio_parser/visitors/script.py index 2abb707..200ec41 100644 --- a/fluxio_parser/visitors/script.py +++ b/fluxio_parser/visitors/script.py @@ -3,15 +3,12 @@ from collections import defaultdict from typing import Any, Dict, Set -from fluxio_parser.exceptions import ( # noqa - assert_supported_operation, - UnsupportedOperation, -) -from fluxio_parser.resource_decorators import RESOURCE_DECORATOR_MAP # noqa -from fluxio_parser.util import parse_options # noqa -from fluxio_parser.visitors.event_processor import EventProcessorVisitor # noqa -from fluxio_parser.visitors.state_machine import StateMachineVisitor # noqa -from fluxio_parser.visitors.task import TaskVisitor # noqa +from fluxio_parser.exceptions import assert_supported_operation, UnsupportedOperation +from fluxio_parser.resource_decorators import RESOURCE_DECORATOR_MAP +from fluxio_parser.util import parse_options +from fluxio_parser.visitors.event_processor import EventProcessorVisitor +from fluxio_parser.visitors.state_machine import StateMachineVisitor +from fluxio_parser.visitors.task import TaskVisitor class ScriptVisitor(ast.NodeVisitor): diff --git a/fluxio_parser/visitors/task.py b/fluxio_parser/visitors/task.py index 5fe6fab..fbba1f7 100644 --- a/fluxio_parser/visitors/task.py +++ b/fluxio_parser/visitors/task.py @@ -11,11 +11,11 @@ class Attribute(NamedTuple): """Data class for a task class attribute schema""" #: Getter function for extracting the value from the AST node - get_value: Callable # noqa + get_value: Callable #: Default value if the attribute is not provided - default_value: Any # noqa + default_value: Any #: Set of allowed values - allowed_values: Set[Union[str, int]] = None # noqa + allowed_values: Set[Union[str, int]] = None # Map of task class attribute name to an attribute schema From e9040ffc41b9d629b38711cba04d8c3070d0f258 Mon Sep 17 00:00:00 2001 From: Jonathan Drake Date: Mon, 29 Mar 2021 17:10:44 -0700 Subject: [PATCH 3/3] nx --- fluxio_parser/states/choice.py | 2 +- fluxio_parser/states/map_.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fluxio_parser/states/choice.py b/fluxio_parser/states/choice.py index 9658e33..f529499 100644 --- a/fluxio_parser/states/choice.py +++ b/fluxio_parser/states/choice.py @@ -222,7 +222,7 @@ class ChoiceState(State): } """ - def __init__(self, state_graph: nx.DiGraph, key: str, ast_node: Any) -> None: + def __init__(self, state_graph: "nx.DiGraph", key: str, ast_node: Any) -> None: """Initializer Args: diff --git a/fluxio_parser/states/map_.py b/fluxio_parser/states/map_.py index e5bac2a..7fecd7c 100644 --- a/fluxio_parser/states/map_.py +++ b/fluxio_parser/states/map_.py @@ -56,7 +56,7 @@ def iterator_function(data): def __init__( self, - state_graph: nx.DiGraph, + state_graph: "nx.DiGraph", key: str, ast_node: Any, iterator: "StateMachineVisitor", # noqa: F821