diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d25e1fb --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,119 @@ +name: CI + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + - v* + +jobs: + linter: + runs-on: ubuntu-latest + strategy: + fail-fast: false + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install linter dependencies + run: | + pip install black + pip install vulture + + - name: Check with black + run: black --check --exclude "(open-simulation-interface|proto2cpp)" . + + - name: Check with vulture + run: vulture *.py tests/ osivalidator/ --min-confidence 100 + + build-osi-validator: + strategy: + fail-fast: false + matrix: + runs-on: [ubuntu-latest] + python-version: [3.6, 3.7, 3.8, 3.9] + + name: "🐍 ${{ matrix.python-version }} • ${{ matrix.runs-on }}" + runs-on: ${{ matrix.runs-on }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Checkout submodules + uses: textbook/git-checkout-submodule-action@master + with: + remote: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install software dependencies + shell: bash + run: | + python -m pip install --upgrade pip + sudo apt-get update -y + sudo apt-get install python3-pip protobuf-compiler + cd open-simulation-interface + pip install . + cd .. + pip install . + + - name: Display infos + shell: bash + run: | + lzma --version + protoc --version + osivalidator --help + + - name: Test lzma compression and decompression + shell: bash + run: | + lzma -z data/small_test.txt + lzma -d data/small_test.txt.lzma + + - name: Run tests + shell: bash + run: python -m unittest discover tests + + - name: Generate rules + shell: bash + run: python open-simulation-interface/rules2yml.py -d rules + + - name: Run validator with different options + shell: bash + run: | + python open-simulation-interface/format/txt2osi.py -d data/small_test.txt + + osivalidator data/small_test.osi + osivalidator data/small_test.osi -p + osivalidator data/small_test.txt + osivalidator data/small_test.txt -p + + osivalidator data/small_test.osi -r rules + osivalidator data/small_test.osi -p -r rules + osivalidator data/small_test.txt -r rules + osivalidator data/small_test.txt -p -r rules + + lzma -z data/small_test.txt + lzma -z data/small_test.osi + + osivalidator data/small_test.osi.lzma + osivalidator data/small_test.osi.lzma -p + osivalidator data/small_test.txt.lzma + osivalidator data/small_test.txt.lzma -p + + osivalidator data/small_test.osi.lzma -r rules + osivalidator data/small_test.osi.lzma -p -r rules + osivalidator data/small_test.txt.lzma -r rules + osivalidator data/small_test.txt.lzma -p -r rules diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index aee475b..0000000 --- a/.travis.yml +++ /dev/null @@ -1,90 +0,0 @@ -# Use distribution Ubuntu 18.04 Bionic -dist: bionic - -# Use python build environment. -language: python - -python: - - "3.6" - -# Handle dependencies in separate directory. -before_install: - - DEPS_DIR="${HOME}/deps" - - mkdir -p "${DEPS_DIR}" - - cd "${DEPS_DIR}" - -# Install necessary packages. -install: - # Install a recent version of the Protobuf - - | - PROTOBUF_URL="https://github.com/google/protobuf/releases/download/v3.2.0/protobuf-python-3.2.0.tar.gz" - if [ ! -f ${DEPS_DIR}/protobuf/install/bin/protoc ] ; then mkdir -p protobuf ; travis_retry wget --no-check-certificate --quiet -O - ${PROTOBUF_URL} | tar --strip-components=1 -xz -C protobuf ; cd protobuf ; ./configure --prefix=${DEPS_DIR}/protobuf/install ; make ; make install ; fi - export PATH=${DEPS_DIR}/protobuf/install/bin:${PATH} - - - sudo apt-get install python3-pip protobuf-compiler - -# Change directory back to default build directory. -before_script: - - cd "${TRAVIS_BUILD_DIR}" - -# Run the build script -script: - # Debug - - which python - - python -m site - - protoc --version - - # Install black code formatter and check if it is applied - - pip install black - - black --check --exclude "(open-simulation-interface|proto2cpp)" . - - # Install vulture and search for dead code with 100% confidence - - pip install vulture - - vulture *.py tests/ osivalidator/ --min-confidence 100 - - # Install osi-validation - - cd open-simulation-interface - - pip install . - - cd .. - - pip install . - - # Generate parsed rules - - python rules2yml.py -d rules - - # Show validator usage - - osivalidator -h - - # Check rule correctness with unittests - - python -m unittest discover tests - - # Convert decompress small_test.txt and convert to small_test.osi - - lzma -d data/small_test.txt.lzma - - python open-simulation-interface/format/txt2osi.py -d data/small_test.txt - - # Run the validator on decompressed data - - osivalidator data/small_test.osi - - osivalidator data/small_test.osi -p - - osivalidator data/small_test.txt - - osivalidator data/small_test.txt -p - - # Run the validator on decompressed data with parsed rules - - osivalidator data/small_test.osi -r rules - - osivalidator data/small_test.osi -p -r rules - - osivalidator data/small_test.txt -r rules - - osivalidator data/small_test.txt -p -r rules - - # Compress *.osi and *.txt with lzma - - lzma -z data/small_test.txt - - lzma -z data/small_test.osi - - # Run validator on a small test with already existing rules - - osivalidator data/small_test.osi.lzma - - osivalidator data/small_test.osi.lzma -p - - osivalidator data/small_test.txt.lzma - - osivalidator data/small_test.txt.lzma -p - - # Run validator on a small test with parsed rules - - osivalidator data/small_test.osi.lzma -r rules - - osivalidator data/small_test.osi.lzma -p -r rules - - osivalidator data/small_test.txt.lzma -r rules - - osivalidator data/small_test.txt.lzma -p -r rules \ No newline at end of file diff --git a/data/small_test.txt b/data/small_test.txt new file mode 100644 index 0000000..863c7f2 --- /dev/null +++ b/data/small_test.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9b7c2acd6ac3a3d222bda3bea7615e9dfd67c7e4b2d36c80034043417ac8a62 +size 6180 diff --git a/data/small_test.txt.lzma b/data/small_test.txt.lzma deleted file mode 100644 index 9568ad3..0000000 --- a/data/small_test.txt.lzma +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:97d4efda5888898a82044e2093ee605bade75290038afaf9a03d7bceecc91d98 -size 442 diff --git a/osivalidator/osi_doxygen_xml.py b/osivalidator/osi_doxygen_xml.py deleted file mode 100644 index 3d782de..0000000 --- a/osivalidator/osi_doxygen_xml.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -This module carry the XML version of Doxygen documentation to parse the rules -into it. -""" - -import os -import glob - -import ruamel.yaml as yaml -import defusedxml.ElementTree as ET - -from doxygen import ConfigParser -from doxygen import Generator - - -class OSIDoxygenXML: - """ - This class creates XML from \*.proto Files Documentation of OSI and can parse the rules from it. - """ - - def __init__(self): - dir_path = os.path.dirname(os.path.realpath(__file__)) - osivalidator_path = os.path.dirname(dir_path) - self.osi_path = os.path.join(osivalidator_path, "open-simulation-interface") - self.osi_doc_path = os.path.join(self.osi_path, "doc") - - proto2cpp_path = os.path.join(osivalidator_path, "proto2cpp") - self.proto2cpp_file_path = os.path.join(proto2cpp_path, "proto2cpp.py") - - def generate_osi_doxygen_xml(self): - """ - Generate the Doxygen XML documentation in the OSI path - """ - - configuration = f""" - PROJECT_NAME = osi-validation - INPUT = {self.osi_path} - OUTPUT_DIRECTORY = {self.osi_doc_path} - EXTENSION_MAPPING = proto=C++ - FILE_PATTERNS = *.proto - INPUT_FILTER = "python {self.proto2cpp_file_path}" - GENERATE_XML = YES - GENERATE_HTML = YES - MAX_DOT_GRAPH_DEPTH = 2 - RECURSIVE = YES - GENERATE_LATEX = NO - XML_PROGRAMLISTING = NO - ALIASES = rules="
"
-        ALIASES               += endrules="
" - """ - - doxyfile_path = os.path.join(self.osi_path, "Doxyfile_validation") - doxyfile = open(doxyfile_path, "w") - doxyfile.write(configuration) - doxyfile.close() - - config_parser = ConfigParser() - config_parser.load_configuration(doxyfile_path) - - doxy_builder = Generator(doxyfile_path) - doxy_builder.build(clean=False, generate_zip=False) - - def get_files(self): - """ - Return the path of the fields in OSI - """ - return glob.glob(os.path.join(self.osi_path, "doc", "xml", "*.xml")) - - def parse_rules(self): - """ - Parse the Doxygen XML documentation to get the rules - """ - xml_files = self.get_files() - - rules = list() - - for xml_file in xml_files: - tree = ET.parse(xml_file) - memberdefs = tree.findall( - "./compounddef/sectiondef/memberdef[@kind='variable']" - ) - for memberdef in memberdefs: - attr_path = memberdef.findtext("definition").split()[-1].split("::") - if attr_path[0] != "osi3": - continue - attr_path.pop(0) - attr_rule = memberdef.findtext( - "./detaileddescription//preformatted[last()]" - ) - if not attr_rule: - continue - - rules_lines = attr_rule.split("\n") - - for line_no, line in enumerate(rules_lines): - if line.find(":") == -1 and line: - rules_lines[line_no] += ":" - - attr_rule = "\n".join(rules_lines) - - try: - dict_rules = yaml.safe_load(attr_rule) - except (yaml.parser.ParserError, yaml.parser.ScannerError) as error: - print(attr_path, attr_rule, error) - else: - rules.append((attr_path, dict_rules)) - return rules - - -if __name__ == "__main__": - osidx = OSIDoxygenXML() - osidx.generate_osi_doxygen_xml() diff --git a/osivalidator/osi_general_validator.py b/osivalidator/osi_general_validator.py index 67930ab..49dec17 100755 --- a/osivalidator/osi_general_validator.py +++ b/osivalidator/osi_general_validator.py @@ -22,7 +22,7 @@ def check_positive_int(value): def command_line_arguments(): - """ Define and handle command line interface """ + """Define and handle command line interface""" dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) diff --git a/osivalidator/osi_id_manager.py b/osivalidator/osi_id_manager.py index 593690a..c5d7d87 100644 --- a/osivalidator/osi_id_manager.py +++ b/osivalidator/osi_id_manager.py @@ -7,8 +7,7 @@ class OSIIDManager: - """Manage the ID of OSI Messages for verification of unicity and references - """ + """Manage the ID of OSI Messages for verification of unicity and references""" def __init__(self, logger): # id => [object1, object2...] diff --git a/osivalidator/osi_rules.py b/osivalidator/osi_rules.py index cb9138e..fa747e2 100644 --- a/osivalidator/osi_rules.py +++ b/osivalidator/osi_rules.py @@ -10,17 +10,15 @@ import ruamel.yaml as yaml -from osivalidator.osi_doxygen_xml import OSIDoxygenXML - class OSIRules: - """ This class collects validation rules """ + """This class collects validation rules""" def __init__(self): self.rules = TypeRulesContainer() def from_yaml_directory(self, path=None): - """ Collect validation rules found in the directory. """ + """Collect validation rules found in the directory.""" if not path: dir_path = dir_path = os.path.dirname(os.path.realpath(__file__)) @@ -36,36 +34,17 @@ def from_yaml_directory(self, path=None): print("Error while reading files OSI-rules. Exiting!") def from_yaml_file(self, path): - """Import from a file - """ + """Import from a file""" rules_file = open(path) self.from_dict(rules_dict=yaml.load(rules_file, Loader=yaml.SafeLoader)) rules_file.close() def from_yaml(self, yaml_content): - """Import from a string - """ + """Import from a string""" self.from_dict(rules_dict=yaml.load(yaml_content, Loader=yaml.SafeLoader)) - def from_xml_doxygen(self): - """Parse the Doxygen XML documentation to get the rules - """ - dox_xml = OSIDoxygenXML() - dox_xml.generate_osi_doxygen_xml() - rules = dox_xml.parse_rules() - - for field_rules_tuple in rules: - message_t_path = field_rules_tuple[0][:-1] - field_name = field_rules_tuple[0][-1] - field_rules = field_rules_tuple[1] - - message_t = self.rules.add_type_from_path(message_t_path) - for field_rule in field_rules: - message_t.add_field(FieldRules(name=field_name, rules=field_rule)) - def get_rules(self): - """Return the rules - """ + """Return the rules""" return self.rules def from_dict(self, rules_dict=None, rules_container=None): @@ -287,8 +266,7 @@ def list_rules(self): return self.rules def get_rule(self, verb): - """Return the rule object for the verb rule_verb in this field. - """ + """Return the rule object for the verb rule_verb in this field.""" return self.rules[verb] def __getitem__(self, verb): diff --git a/osivalidator/osi_rules_implementations.py b/osivalidator/osi_rules_implementations.py index 120c47b..79eb94e 100644 --- a/osivalidator/osi_rules_implementations.py +++ b/osivalidator/osi_rules_implementations.py @@ -12,8 +12,7 @@ def add_default_rules_to_subfields(message, type_rules): - """Add default rules to fields of message fields (subfields) - """ + """Add default rules to fields of message fields (subfields)""" for descriptor in message.all_field_descriptors: field_rules = ( type_rules.get_field(descriptor.name) @@ -46,15 +45,13 @@ def pre_check(func): def repeated_selector(func): - """Decorator for selector-rules that take - """ + """Decorator for selector-rules that take""" func.repeated_selector = True return func def rule_implementation(func): - """Decorator to label rules method implementations - """ + """Decorator to label rules method implementations""" func.is_rule = True @wraps(func) diff --git a/osivalidator/osi_trace.py b/osivalidator/osi_trace.py index 19111fa..0cbe81c 100644 --- a/osivalidator/osi_trace.py +++ b/osivalidator/osi_trace.py @@ -320,7 +320,7 @@ def get_messages_in_index_range(self, begin, end): for rel_index, rel_message_offset in enumerate(rel_message_offsets): rel_begin = rel_message_offset + self._int_length rel_end = ( - rel_message_offsets[rel_index + 1] - self._int_length + rel_message_offsets[rel_index + 1] if rel_index + 1 < len(rel_message_offsets) else message_sequence_len ) diff --git a/osivalidator/osi_validator_logger.py b/osivalidator/osi_validator_logger.py index bfbb097..dd5840f 100644 --- a/osivalidator/osi_validator_logger.py +++ b/osivalidator/osi_validator_logger.py @@ -109,8 +109,7 @@ def create_database(self, timestamp, output_path): cursor.close() def init_logging_storage(self, files, output_path): - """Initialize (create or set handler) for the specified logging storage - """ + """Initialize (create or set handler) for the specified logging storage""" timestamp = time.time() if files: self._init_logging_to_files(timestamp, output_path) diff --git a/rules2yml.py b/rules2yml.py index 4f35731..6af2cec 100644 --- a/rules2yml.py +++ b/rules2yml.py @@ -7,7 +7,7 @@ def command_line_arguments(): - """ Define and handle command line interface """ + """Define and handle command line interface""" dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) diff --git a/setup.py b/setup.py index 02317db..6365c15 100644 --- a/setup.py +++ b/setup.py @@ -45,12 +45,11 @@ "sphinx_rtd_theme", "recommonmark", "open-simulation-interface", - "doxygen-interface", "defusedxml", "colorama", "tabulate", "progress", - "protobuf==3.9.1", + "protobuf", ], dependency_links=[ "git+https://github.com/OpenSimulationInterface/" diff --git a/tests/test_trace.py b/tests/test_trace.py index 42dc809..49ab150 100644 --- a/tests/test_trace.py +++ b/tests/test_trace.py @@ -16,9 +16,9 @@ def setUp(self): self.osi_nobuffer = OSITrace(buffer_size=0) self.txt.from_file( - path="data/small_test.txt.lzma", type_name="SensorView", + path="data/small_test.txt", + type_name="SensorView", ) - subprocess.call(["lzma", "-d", "data/small_test.txt.lzma"]) subprocess.call( [ "python3", @@ -46,6 +46,7 @@ def tearDown(self): subprocess.call(["rm", "data/small_test.osi.lzma"]) subprocess.call(["rm", "data/small_test.osi"]) + subprocess.call(["lzma", "-d", "data/small_test.txt.lzma"]) def test_get_messages_in_index_range(self): """Test getting messages in range""" diff --git a/tests/test_validation_rules.py b/tests/test_validation_rules.py index b2a99e0..61ca9df 100644 --- a/tests/test_validation_rules.py +++ b/tests/test_validation_rules.py @@ -38,7 +38,7 @@ def test_add_type_from_path(self): self.assertEqual(path.path, typecontainer.path.path) def test_parse_yaml(self): - """ Test the YAML parsing""" + """Test the YAML parsing""" raw = """ HostVehicleData: location: diff --git a/tests/tests_rules/test_is_greater_than.py b/tests/tests_rules/test_is_greater_than.py index 8b1031a..a15e77c 100644 --- a/tests/tests_rules/test_is_greater_than.py +++ b/tests/tests_rules/test_is_greater_than.py @@ -8,7 +8,7 @@ class TestIsGreaterThan(unittest.TestCase): - """ Test for rule is_greater_than """ + """Test for rule is_greater_than""" def setUp(self): self.FRC = OSIRulesChecker() diff --git a/tests/tests_rules/test_is_greater_than_or_equal_to.py b/tests/tests_rules/test_is_greater_than_or_equal_to.py index b97d2ff..38c803d 100644 --- a/tests/tests_rules/test_is_greater_than_or_equal_to.py +++ b/tests/tests_rules/test_is_greater_than_or_equal_to.py @@ -8,7 +8,7 @@ class TestIsGreaterThanOrEqualTo(unittest.TestCase): - """ Test for rule is_greater_than_or_equal_to """ + """Test for rule is_greater_than_or_equal_to""" def setUp(self): self.FRC = OSIRulesChecker()