prepare release 0.9.3 (#543)

Author: ebroecker

Diff for: requirements.test.py2.txt

@@ -16,7 +16,7 @@ tox==3.2.1
 typing==3.6.6
 virtualenv==16.0.0
 xlwt==1.3.0
-xlrd==1.1.0
+xlrd==1.2.0
 click==7.0
-lxml==4.5.2
+lxml==4.6.2
 enum34==1.1.10

Diff for: requirements.test.py3.txt

@@ -16,8 +16,8 @@ tox==3.2.1
 typing==3.6.6; python_version < '3.5'
 virtualenv==16.0.0
 xlwt==1.3.0
-xlrd==1.1.0
-lxml==4.5.2
+xlrd==1.2.0
+lxml==4.6.2
 click==7.0
 xlsxwriter==1.2.8
 pyaml==20.4.0

Diff for: requirements.tox.txt

@@ -8,3 +8,4 @@ typing==3.6.6; python_version < '3.5'
 virtualenv==16.4.1
 click==7.0
 enum34==1.1.10; python_version <= '2.7'
+xlrd==1.2.0

Diff for: requirements.txt

@@ -1,7 +1,7 @@
 click
 lxml
 xlwt
-xlrd
+xlrd==1.2.0
 xlsxwriter
 pyyaml
 future

Diff for: src/canmatrix/canmatrix.py

@@ -1351,35 +1351,18 @@ def unpack(self, data, report_error=True):
 
             return returnDict
 
-    def _has_sub_multiplexer(self, parent_multiplexer_name):
+    def _get_sub_multiplexer(self, parent_multiplexer_name, parent_multiplexer_value):
         """
-        check if any sub-multiplexer in frame
-        used for complex-multiplexed frame decoding
-
-        :param parent_multiplexer_name: string with name of parent multiplexer
-        :return: True or False
-        """
-        for signal in self.signals:
-            if signal.is_multiplexer and signal.muxer_for_signal == parent_multiplexer_name:
-                return True
-        return False
-
-    def _get_sub_multiplexer(self, parent_multiplexer_name, parent_multiplexer_value, decoded):
-        """
-        get any sub-multiplexer in frame for decoded data
-        return multiplexers name and value
-        used for complex-multiplexed frame decoding
+        get any sub-multiplexer in frame used
+        for complex-multiplexed frame decoding
 
         :param parent_multiplexer_name: string with name of parent multiplexer
         :param parent_multiplexer_value: raw_value (int) of parent multiplexer
-        :param decoded: OrderedDictionary which is returned from canmatrix.Frame.unpack
-        :return: muxer_name and muxer_value
+        :return: muxer signal or None
         """
         for signal in self.signals:
             if signal.is_multiplexer and signal.muxer_for_signal == parent_multiplexer_name and signal.multiplexer_value_in_range(parent_multiplexer_value):
-                muxer_value = decoded[signal.name].raw_value
-                muxer_name = signal.name
-                return muxer_name, muxer_value
+                return signal
 
     def _filter_signals_for_multiplexer(self, multiplexer_name, multiplexer_value):
         """
@@ -1417,11 +1400,15 @@ def decode(self, data):
             multiplex_name = None
             multiplex_value = None
 
-            while self._has_sub_multiplexer(multiplex_name):
-                multiplex_name, multiplex_value = self._get_sub_multiplexer(multiplex_name, multiplex_value, decoded)
-                decoded_values[multiplex_name] = decoded[multiplex_name]
+            sub_multiplexer = self._get_sub_multiplexer(multiplex_name, multiplex_value)
+            while sub_multiplexer is not None:
+                multiplex_name = sub_multiplexer.name
+                multiplex_signal = decoded_values[multiplex_name] = decoded[multiplex_name]
+                multiplex_value = multiplex_signal.raw_value
                 filtered_signals += self._filter_signals_for_multiplexer(multiplex_name, multiplex_value)
 
+                sub_multiplexer = self._get_sub_multiplexer(multiplex_name, multiplex_value)
+
             for signal in filtered_signals:
                 decoded_values[signal.name] = decoded[signal.name]
             return decoded_values

Diff for: src/canmatrix/formats/__init__.py

@@ -27,7 +27,7 @@
         importlib.import_module("canmatrix.formats." + module)
         loadedFormats.append(module)
     except ImportError:
-        logger.exception("%s is not supported", module)
+        logger.warning("%s is not supported", module)
 
 for loadedModule in loadedFormats:
     supportedFormats[loadedModule] = []

Diff for: src/canmatrix/formats/arxml.py

@@ -1732,6 +1732,7 @@ def decode_ethernet_helper(ea, float_factory):
                 pdu_sig_mapping = ea.findall("I-SIGNAL-TO-I-PDU-MAPPING", ipdu)
 
                 get_signals(pdu_sig_mapping, target_frame, ea, None, float_factory)
+                target_frame.update_receiver()
                 db.add_frame(target_frame)
     return found_matrixes
 
@@ -1750,14 +1751,14 @@ def decode_flexray_helper(ea, float_factory):
             found_matrixes[channel_name] = db
 
             frames = ea.findall("FLEXRAY-FRAME-TRIGGERING", pc)
-            for frame in frames:
+            for frame_element in frames:
                 frame_counter += 1
-                slot_id = int(ea.get_child(frame, "SLOT-ID").text)
-                base_cycle = ea.get_child(frame, "BASE-CYCLE").text
-                ipdu_triggerings = ea.get_children(frame, "I-PDU-TRIGGERING")
-                frame_repetition_cycle = ea.find_children_by_path(frame, "CYCLE-REPETITION/CYCLE-REPETITION")[0].text
-                network_endpoints = pc.findall('.//' + ns + "NETWORK-ENDPOINT")
-                frame_size = int(ea.find_children_by_path(frame, "FRAME/FRAME-LENGTH")[0].text)
+                slot_id = int(ea.get_child(frame_element, "SLOT-ID").text)
+                base_cycle = ea.get_child(frame_element, "BASE-CYCLE").text
+                ipdu_triggerings = ea.get_children(frame_element, "I-PDU-TRIGGERING")
+                frame_repetition_cycle = ea.find_children_by_path(frame_element, "CYCLE-REPETITION/CYCLE-REPETITION")[0].text
+                network_endpoints = pc.findall('.//' + ea.ns + "NETWORK-ENDPOINT")
+                frame_size = int(ea.find_children_by_path(frame_element, "FRAME/FRAME-LENGTH")[0].text)
                 frame = canmatrix.Frame(size = frame_size, arbitration_id = frame_counter)
                 frame.slot_id = slot_id
                 frame.base_cycle = base_cycle
@@ -1888,7 +1889,10 @@ def decode_can_helper(ea, float_factory, ignore_cluster_info):
                     sig_value_hash[sig.name] = 0
             frame_data = frame.encode(sig_value_hash)
             frame.add_attribute("GenMsgStartValue", "".join(["%02x" % x for x in frame_data]))
+            frame.update_receiver()
         found_matrixes[bus_name] = db
+
+
     return found_matrixes
 
 def load(file, **options):

Diff for: src/canmatrix/formats/dbc.py

@@ -35,7 +35,7 @@
 from builtins import *
 
 import canmatrix
-
+import canmatrix.utils
 logger = logging.getLogger(__name__)
 
 
@@ -755,13 +755,15 @@ def add_frame_by_id(new_frame):  # type: (canmatrix.Frame) -> None
                 if temp:
                     frame_id = temp.group(1)
                     signal_name = temp.group(2)
-                    temp_list = temp.group(3).split('"')
+                    temp_list = list(canmatrix.utils.escape_aware_split(temp.group(3), '"'))
+
                     if frame_id.isnumeric():  # value for Frame
                         try:
                             frame = get_frame_by_id(canmatrix.ArbitrationId.from_compound_integer(int(frame_id)))
                             sg = frame.signal_by_name(signal_name)
                             for i in range(math.floor(len(temp_list) / 2)):
                                 val = temp_list[i * 2 + 1]
+                                val = val.replace('\\"', '"')
                                 if sg:
                                     sg.add_values(temp_list[i * 2], val)
                         except:
@@ -930,7 +932,7 @@ def add_frame_by_id(new_frame):  # type: (canmatrix.Frame) -> None
             ecu.name = ecu.attributes.get("SystemNodeLongSymbol")[1:-1]
             ecu.del_attribute("SystemNodeLongSymbol")
     for frame in db.frames:
-        frame.cycle_time = int(frame.attributes.get("GenMsgCycleTime", 0))
+        frame.cycle_time = int(float(frame.attributes.get("GenMsgCycleTime", 0)))
         if frame.attributes.get("SystemMessageLongSymbol", None) is not None:
             frame.name = frame.attributes.get("SystemMessageLongSymbol")[1:-1]
             frame.del_attribute("SystemMessageLongSymbol")

Diff for: src/canmatrix/formats/json.py

@@ -93,6 +93,7 @@ def dump(db, f, **options):
             symbolic_frame = {"name": frame.name,
                               "id": int(frame.arbitration_id.id),
                               "is_extended_frame": frame.arbitration_id.extended,
+                              "is_fd": frame.is_fd,
                               "signals": symbolic_signals}
             frame_attributes = {
                 attr: frame.attribute(attr)
@@ -146,6 +147,7 @@ def dump(db, f, **options):
                 {"name": frame.name,
                  "id": int(frame.arbitration_id.id),
                  "is_extended_frame": frame.arbitration_id.extended,
+                 "is_fd": frame.is_fd,
                  "signals": symbolic_signals,
                  "attributes": frame_attributes,
                  "comment": frame.comment,

Diff for: src/canmatrix/formats/sym.py

@@ -442,7 +442,10 @@ class Mode(object):
                     is_float = False
                     is_ascii = False
                     enumeration = None
-                    if index_offset != 1:
+
+                    if tmp_mux == "Mux":
+                        is_signed = False
+                    elif index_offset != 1 :
                         is_signed = True
                     else:
                         is_signed = False

Diff for: src/canmatrix/tests/test_dbc.py

@@ -478,4 +478,31 @@ def test_missing_space():
     matrix = canmatrix.formats.dbc.load(dbc, dbcImportEncoding="utf8")
     assert matrix.frames[0].signals[0].name == "sig1"
 
+def test_escaped_quotes():
+    dbc = io.BytesIO(textwrap.dedent(r'''
+        BO_ 17 Frame_1: 8 Vector__XXX
+        SG_ Signal : 0|8@1-(1,0)[0|0] "" Vector__XXX
+        
+        VAL_ 17 Signal 0 "zero" 1 "one " 2 "string with \"escaped\" double quotes";
+        ''').encode('utf-8'))
+    matrix = canmatrix.formats.dbc.load(dbc, dbcImportEncoding="utf8")
+    assert matrix.frames[0].signals[0].values[2] == r'string with "escaped" double quotes'
+
+
+def test_float_cycle_time():
+    dbc = io.BytesIO(textwrap.dedent(u'''\
+        BO_ 17 Frame_1: 8 Vector__XXX
+        SG_ sig2 : 8|8@1- (1,0) [0|0] "" Vector__XXX
+        SG_ sig1 : 0|8@1- (1,0) [0|0] "" Vector__XXX
+
+
+        BA_DEF_ BO_  "GenMsgCycleTime" INT 10 3600000;
+        BA_ "GenMsgCycleTime" BO_ 17 100.0;
+''').encode('utf-8'))
+
+
+    matrix = canmatrix.formats.dbc.load(dbc, dbcImportEncoding="utf8")
+
+    assert matrix.frames[0].cycle_time == 100
+
 

Diff for: src/canmatrix/tests/test_json.py

@@ -88,6 +88,7 @@ def test_import_min_max():
                 "comment": "",
                 "id": 10,
                 "is_extended_frame": false,
+                "is_fd": false,
                 "length": 6,
                 "name": "test_frame",
                 "signals": [
@@ -122,6 +123,7 @@ def test_import_native():
                 "comment": "",
                 "id": 10,
                 "is_extended_frame": false,
+                "is_fd": false,
                 "length": 6,
                 "name": "test_frame",
                 "signals": [
@@ -167,6 +169,7 @@ def test_import_export_enums():
                 "comment": "",
                 "id": 10,
                 "is_extended_frame": false,
+                "is_fd": false,
                 "length": 6,
                 "name": "test_frame",
                 "signals": [
@@ -218,3 +221,4 @@ def test_export_all_native():
     assert (data['messages'][0]['signals'][0]['max'] == 42)
     assert (data['messages'][0]['signals'][0]['factor'] == 0.123)
     assert (data['messages'][0]['signals'][0]['offset'] == 1)
+    assert (data['messages'][0]['is_fd'] is False)

Diff for: src/canmatrix/tests/test_sym.py

@@ -3,6 +3,7 @@
 import sys
 import textwrap
 from itertools import chain
+from pprint import pprint
 
 import pytest
 
@@ -353,4 +354,47 @@ def test_types_read():
                                                             "Double",
                                                             "Float", ])
 
+@pytest.mark.parametrize(
+    'var_name,data,raw_value',
+    (
+        ('VarMux1',   bytearray([1,  12,  0,  0,  0,  0,  0,  0]), 12),
+        ('VarMux2',   bytearray([2,  0,  0,  0,  23,  0,  0,  0]), 23),
+        ('VarMux200', bytearray([200,  0,  0,  0,  0,  0,  34,  0]), 34),
+    )
+)
+def test_mux_decode(var_name,data,raw_value):
+    f = io.BytesIO('''\
+    FormatVersion=5.0 // Do not edit this line!
+    Title="Types Test"
+
+FormatVersion=5.0 // Do not edit this line!
+Title="Test Symbols File"
+
+{SENDRECEIVE}
+
+[MuxTestFrame]
+ID=002h
+DLC=8
+Mux=Mux1 0,8 1 
+Var=VarMux1 unsigned 8,8
+
+[MuxTestFrame]
+DLC=8
+Mux=Mux2 0,8 2 
+Var=VarMux2 unsigned 32,8
+
+[MuxTestFrame]
+DLC=8
+Mux=Mux200 0,8 C8h
+Var=VarMux200 unsigned 48,8
+    '''.encode('utf-8'),
+                   )
+
+    matrix = canmatrix.formats.sym.load(f)
+    # Check no errors loading the matrix
+    assert matrix.load_errors == []
 
+    frame = matrix.frame_by_name("MuxTestFrame")
+    r = frame.decode(data)
+    assert var_name in r.keys(), "Signal {}, not decoded. Only : {}".format(var_name, ','.join(r for r in r.keys()))
+    assert r[var_name].raw_value == raw_value

Diff for: src/canmatrix/utils.py

@@ -20,6 +20,26 @@ def quote_aware_space_split(in_line):  # type: (str) -> typing.List[str]
     return [item.decode('utf-8') for item in shlex.split(in_line.strip().encode('utf-8'))]
 
 
+# https://stackoverflow.com/questions/18092354/python-split-string-without-splitting-escaped-character
+def escape_aware_split(string, delimiter):
+    if len(delimiter) != 1:
+        raise ValueError('Invalid delimiter: ' + delimiter)
+    ln = len(string)
+    i = 0
+    j = 0
+    while j < ln:
+        if string[j] == '\\':
+            if j + 1 >= ln:
+                yield string[i:j]
+                return
+            j += 1
+        elif string[j] == delimiter:
+            yield string[i:j]
+            i = j + 1
+        j += 1
+    yield string[i:j]
+
+
 def quote_aware_comma_split(string):  # type: (str) -> typing.List[str]
     """
     Split a string containing comma separated list of fields.

Diff for: test/test.py

@@ -44,6 +44,9 @@ def run_tests():
     export_types.sort()
     # TODO: support testing of xlsx
     # export_types.remove('xlsx')
+    if "xlsx" in import_types:
+        # todo issue #541
+        import_types.remove("xlsx")
     if "fibex" in export_types:
         export_types.remove('fibex')