Updated main.py, inputs.py and utils.py to incorporate support for .MFC, .MFE, .MFM files (data compression and/or data encryption)

Author: Martin

README.md

@@ -63,6 +63,14 @@ Antoher approach is to use event based triggers, e.g. via AWS Lambda functions.
 ---
 ## Other practical information
 
+### Regarding encrypted log files
+If you need to handle encrypted log files, you can provide a passwords dictionary object with similar structure as the `passwords.json` file used in the CANedge MF4 converters. The object can be provided e.g. as below (or via environmental variables):
+
+```
+pw = {"default": "password"} 			# hardcoded  
+pw = json.load(open("passwords.json"))	# from local JSON file 
+```
+
 ### Change timestamps 
 If you wish to test the script using old data, you can change the timestamps so that the data is 'rebaselined' to today, minus an offset number of days. This is useful e.g. if you want to use the InfluxDB Cloud Starter, which will delete data that is older than 30 days. To rebaseline your data to start today minus 2 days, simply add `days_offset=2` in the `ProcessData` initialization. 
 

build_layers.py

@@ -0,0 +1,80 @@
+# requires Docker to be running
+
+import os, json, sys
+import subprocess
+
+# specify base details and region list
+layer_name = "css-dashboard-writer"
+layer_description = "CSS Electronics dashboard-writer script dependencies for use in AWS Lambda functions"
+csv_path = "dashboard-writer/aws_lambda_example/lambda_layer_arns.csv"
+run_req_build = False
+
+regions = [
+    "ap-northeast-1",
+    "ap-northeast-2",
+    "ap-south-1",
+    "ap-southeast-1",
+    "ap-southeast-2",
+    "ca-central-1",
+    "eu-central-1",
+    "eu-north-1",
+    "eu-west-1",
+    "eu-west-2",
+    "eu-west-3",
+    "sa-east-1",
+    "us-east-1",
+    "us-east-2",
+    "us-west-1",
+    "us-west-2",
+]
+
+# create zip with requirements.txt dependencies
+if run_req_build:
+    os.system("rmdir /S/Q python")
+    os.system("mkdir python\lib\python3.7\site-packages")
+    os.system(
+        'docker run -v "%cd%":/var/task "lambci/lambda:build-python3.7" /bin/sh -c "pip install -r requirements.txt -t python/lib/python3.7/site-packages/; exit"'
+    )
+    os.system("rmdir /S/Q python\\lib\\python3.7\\site-packages\\botocore")
+    os.system("zip -r dashboard-writer.zip python")
+
+# for each region, publish AWS layer with build zip
+region_arn_list = []
+
+print("we get here")
+
+for region in regions:
+    print(f"executing region {region}")
+
+    # create the layers
+    arn_output = subprocess.check_output(
+        f'aws lambda publish-layer-version --region {region} --layer-name {layer_name} --description "{layer_description}" --cli-connect-timeout 6000 --license-info "MIT" --zip-file "fileb://dashboard-writer.zip" --compatible-runtimes python3.7',
+        shell=True,
+    ).decode("utf-8")
+
+    print("We build the layer", arn_output)
+
+    version = int(json.loads(arn_output)["Version"])
+
+    # make them public
+    make_public = subprocess.check_output(
+        f"aws lambda add-layer-version-permission --layer-name {layer_name} --version-number {version} --statement-id allAccountsExample --principal * --action lambda:GetLayerVersion --region {region}",
+        shell=True,
+    )
+
+    print("Build layer:", arn_output)
+
+    print("Make public:", make_public)
+
+    arn = str(json.loads(arn_output)["LayerVersionArn"])
+    region_arn = f"{region},{arn}\n"
+    region_arn_list.append(region_arn)
+
+# write data to CSV
+output_file = open(csv_path, "w")
+for region_arn in region_arn_list:
+    output_file.write(region_arn)
+
+output_file.close()
+
+print(f"Completed writing {len(region_arn_list)} out of {len(regions)} to CSV {csv_path}")

dashboard-writer/inputs.py

@@ -11,6 +11,9 @@
 # Syntax: Local: ["folder/device_id"] | S3: ["bucket/device_id"]
 devices = ["LOG/958D2219"]
 
+# if you're using data encryption, you can pass the password below
+pw = {"default": "password"}
+
 
 # -----------------------------------------------
 # specify DBC paths and a list of signals to process ([]: include all signals)

dashboard-writer/main.py

@@ -7,15 +7,15 @@
 start_times = influx.get_start_times(inp.devices, inp.default_start, inp.dynamic)
 
 # setup filesystem (local/S3), load DBC files and list log files for processing
-fs = setup_fs(inp.s3, inp.key, inp.secret, inp.endpoint)
+fs = setup_fs(inp.s3, inp.key, inp.secret, inp.endpoint, inp.pw)
 db_list = load_dbc_files(inp.dbc_paths)
-log_files = list_log_files(fs, inp.devices, start_times)
+log_files = list_log_files(fs, inp.devices, start_times, inp.pw)
 
 # process log files and write extracted signals to InfluxDB
 proc = ProcessData(fs, db_list, inp.signals)
 
 for log_file in log_files:
-    df_raw, device_id = proc.get_raw_data(log_file)
+    df_raw, device_id = proc.get_raw_data(log_file, inp.pw)
     df_phys = proc.extract_phys(df_raw)
 
     proc.print_log_summary(device_id, log_file, df_phys)

dashboard-writer/utils.py

@@ -1,4 +1,4 @@
-def setup_fs(s3, key="", secret="", endpoint="", cert=""):
+def setup_fs(s3, key="", secret="", endpoint="", cert="", passwords={}):
     """Given a boolean specifying whether to use local disk or S3, setup filesystem
     Syntax examples: AWS (http://s3.us-east-2.amazonaws.com), MinIO (http://192.168.0.1:9000)
     The cert input is relevant if you're using MinIO with TLS enabled, for specifying the path to the certficiate.
@@ -27,7 +27,7 @@ def setup_fs(s3, key="", secret="", endpoint="", cert=""):
         import canedge_browser
 
         base_path = Path(__file__).parent
-        fs = canedge_browser.LocalFileSystem(base_path=base_path)
+        fs = canedge_browser.LocalFileSystem(base_path=base_path, passwords=passwords)
 
     return fs
 
@@ -48,7 +48,7 @@ def load_dbc_files(dbc_paths):
 
 
 # -----------------------------------------------
-def list_log_files(fs, devices, start_times, verbose=True):
+def list_log_files(fs, devices, start_times, verbose=True, passwords={}):
     """Given a list of device paths, list log files from specified filesystem.
     Data is loaded based on the list of start datetimes
     """
@@ -59,7 +59,7 @@ def list_log_files(fs, devices, start_times, verbose=True):
     if len(start_times):
         for idx, device in enumerate(devices):
             start = start_times[idx]
-            log_files_device = canedge_browser.get_log_files(fs, [device], start_date=start)
+            log_files_device = canedge_browser.get_log_files(fs, [device], start_date=start, passwords=passwords)
             log_files.extend(log_files_device)
 
     if verbose:
@@ -180,14 +180,14 @@ def filter_signals(self, df_phys):
 
         return df_phys
 
-    def get_raw_data(self, log_file, lin=False):
+    def get_raw_data(self, log_file, lin=False, passwords={}):
         """Extract a df of raw data and device ID from log file.
         Optionally include LIN bus data by setting lin=True
         """
         import mdf_iter
 
         with self.fs.open(log_file, "rb") as handle:
-            mdf_file = mdf_iter.MdfFile(handle)
+            mdf_file = mdf_iter.MdfFile(handle, passwords=passwords)
             device_id = self.get_device_id(mdf_file)
 
             if lin:

requirements.txt

@@ -6,7 +6,7 @@ attrs==21.2.0
 bitstruct==8.11.1
 botocore==1.20.49
 can-decoder>=0.1.3
-canedge-browser>=0.0.7
+canedge-browser>=0.0.8
 canmatrix==0.9.1
 certifi==2021.5.30
 chardet==4.0.0
@@ -19,7 +19,7 @@ importlib-metadata==4.5.0
 influxdb-client==1.10.0
 J1939-PGN==0.4
 jmespath==0.10.0
-mdf-iter>=0.0.4
+mdf-iter>=0.0.6
 multidict==5.1.0
 numpy==1.20.3
 pandas==1.2.4