v2020.4.10 release

CHANGES.md

@@ -1,5 +1,10 @@
 
 
+v2020.4.10
+----------
+* Change the paused column in the summary data to be int type
+* Reduce actual peak memory usage and compute time - Thanks to Linda Hung
+
 v2020.4.9
 ---------
 * Reduce memory and reduce warnings - Thanks to Linda Hung

beep/__init__.py

@@ -21,7 +21,7 @@
 # Versioning.  The python code version is frequently tagged
 # with a commit hash from the repo, which is supplied via
 # an environment variable by the integration build procedure
-__version__ = "2020.4.9"
+__version__ = "2020.4.10"
 VERSION_TAG = os.environ.get("BEEP_VERSION_TAG")
 if VERSION_TAG is not None:
     __version__ = '-'.join([__version__, VERSION_TAG])

beep/config.py

@@ -12,7 +12,7 @@
             'streams': ['CloudWatch']
         },
         'kinesis': {
-            'stream': 'local/beep/eventstream'
+            'stream': 'stage/beep/eventstream/stage'
         }
     },
 

beep/tests/test_files/2017-06-30_2C-10per_6C_CH10_full_model_multi_features.json

@@ -1 +1 @@
-{"@module": "beep.featurize", "@class": "DegradationPredictor", "name": "full_model", "X": {"discharge_capacity_cycle_2": [1.06918], "max_discharge_capacity_difference": [0.8851966], "discharge_capacity_cycle_100": [0.96991605], "integrated_time_temperature_cycles_1:100": [222726.94174602136], "charge_time_cycles_1:5": [612.3784400000004], "abs_min_discharge_capacity_difference_cycles_2:100": [-0.7348436256906956], "abs_mean_discharge_capacity_difference_cycles_2:100": [-1.1053494691021108], "abs_variance_discharge_capacity_difference_cycles_2:100": [-2.221331230186896], "abs_skew_discharge_capacity_difference_cycles_2:100": [-0.833268529824358], "abs_kurtosis_discharge_capacity_difference_cycles_2:100": [0.25659684973168734], "abs_first_discharge_capacity_difference_cycles_2:100": [-0.9555498187487086], "max_temperature_cycles_1:100": [38.543324], "min_temperature_cycles_1:100": [27.851921], "slope_discharge_capacity_cycle_number_2:100": [-0.0010792954789115617], "intercept_discharge_capacity_cycle_number_2:100": [1.0836943592991142], "slope_discharge_capacity_cycle_number_91:100": [-0.0013609268484848626], "intercept_discharge_capacity_cycle_number_91:100": [1.1050065801818196], "min_internal_resistance_cycles_2:100": [0.016267318], "internal_resistance_cycle_2": [0.017852232], "internal_resistance_difference_cycles_2:100": [-0.0009825359999999991]}, "feature_labels": ["discharge_capacity_cycle_2", "max_discharge_capacity_difference", "discharge_capacity_cycle_100", "integrated_time_temperature_cycles_1:100", "charge_time_cycles_1:5", "abs_min_discharge_capacity_difference_cycles_2:100", "abs_mean_discharge_capacity_difference_cycles_2:100", "abs_variance_discharge_capacity_difference_cycles_2:100", "abs_skew_discharge_capacity_difference_cycles_2:100", "abs_kurtosis_discharge_capacity_difference_cycles_2:100", "abs_first_discharge_capacity_difference_cycles_2:100", "max_temperature_cycles_1:100", "min_temperature_cycles_1:100", "slope_discharge_capacity_cycle_number_2:100", "intercept_discharge_capacity_cycle_number_2:100", "slope_discharge_capacity_cycle_number_91:100", "intercept_discharge_capacity_cycle_number_91:100", "min_internal_resistance_cycles_2:100", "internal_resistance_cycle_2", "internal_resistance_difference_cycles_2:100"], "predict_only": false, "prediction_type": "multi", "nominal_capacity": 1.0628421000000001, "y": {"capacity_0.98": [44], "capacity_0.95": [71], "capacity_0.92": [94], "capacity_0.89": [114], "capacity_0.86": [132], "capacity_0.83": [147], "capacity_0.8": [161]}, "@version": "2020.3.31"}
+{"@module": "beep.featurize", "@class": "DegradationPredictor", "name": "full_model", "X": {"discharge_capacity_cycle_2": [1.06918], "max_discharge_capacity_difference": [0.8851966], "discharge_capacity_cycle_100": [0.96991605], "integrated_time_temperature_cycles_1:100": [222726.94174602136], "charge_time_cycles_1:5": [612.3784400000004], "abs_min_discharge_capacity_difference_cycles_2:100": [-0.7348436256906956], "abs_mean_discharge_capacity_difference_cycles_2:100": [-1.1053494691021108], "abs_variance_discharge_capacity_difference_cycles_2:100": [-2.221331230186896], "abs_skew_discharge_capacity_difference_cycles_2:100": [-0.833268529824358], "abs_kurtosis_discharge_capacity_difference_cycles_2:100": [0.25659684973168734], "abs_first_discharge_capacity_difference_cycles_2:100": [-0.9555498187487086], "max_temperature_cycles_1:100": [38.543324], "min_temperature_cycles_1:100": [27.851921], "slope_discharge_capacity_cycle_number_2:100": [-0.0010792954789115617], "intercept_discharge_capacity_cycle_number_2:100": [1.0836943592991142], "slope_discharge_capacity_cycle_number_91:100": [-0.0013609268484848626], "intercept_discharge_capacity_cycle_number_91:100": [1.1050065801818196], "min_internal_resistance_cycles_2:100": [0.016267318], "internal_resistance_cycle_2": [0.017852232], "internal_resistance_difference_cycles_2:100": [-0.0009825359999999991]}, "feature_labels": ["discharge_capacity_cycle_2", "max_discharge_capacity_difference", "discharge_capacity_cycle_100", "integrated_time_temperature_cycles_1:100", "charge_time_cycles_1:5", "abs_min_discharge_capacity_difference_cycles_2:100", "abs_mean_discharge_capacity_difference_cycles_2:100", "abs_variance_discharge_capacity_difference_cycles_2:100", "abs_skew_discharge_capacity_difference_cycles_2:100", "abs_kurtosis_discharge_capacity_difference_cycles_2:100", "abs_first_discharge_capacity_difference_cycles_2:100", "max_temperature_cycles_1:100", "min_temperature_cycles_1:100", "slope_discharge_capacity_cycle_number_2:100", "intercept_discharge_capacity_cycle_number_2:100", "slope_discharge_capacity_cycle_number_91:100", "intercept_discharge_capacity_cycle_number_91:100", "min_internal_resistance_cycles_2:100", "internal_resistance_cycle_2", "internal_resistance_difference_cycles_2:100"], "predict_only": false, "prediction_type": "multi", "nominal_capacity": 1.0628421000000001, "y": {"capacity_0.98": [44], "capacity_0.95": [71], "capacity_0.92": [94], "capacity_0.89": [114], "capacity_0.86": [132], "capacity_0.83": [147], "capacity_0.8": [161]}, "@version": "2020.4.5"}

beep/utils/events.py

@@ -69,7 +69,7 @@ def __init__(self,
             self.kinesis = boto3.client('kinesis', region_name='us-west-2')
 
         if self.mode == 'test':
-            self.stream = get_secret(config[ENVIRONMENT]['kinesis']['stream'])['streamName']
+            self.stream = get_secret(config['test']['kinesis']['stream'])['streamName']
             self.kinesis = boto3.client('kinesis', region_name='us-west-2')
 
         if self.mode == 'events_off':

beep/utils/retrigger.py

@@ -14,19 +14,22 @@
     --version                Show version
     --mode <mode>            Mode to run in [default: 'test']
     --s3_prefix <s3_prefix>  Prefix to use [default: 'd3Batt/raw/arbin']
+    --s3_output <s3_output>  Output prefix to use [default: 'd3Batt/structure']
 
 """
 
 import time
 import datetime
+import pytz
 from dateutil.tz import tzutc
 import boto3
 import collections
 import ast
 from docopt import docopt
 from beep.utils import KinesisEvents
 
-S3_BUCKET = "beep-input-data"
+S3_BUCKET_IN = "beep-input-data-stage"
+S3_BUCKET_OUT = "beep-output-data-stage"
 
 
 class DotDict(collections.OrderedDict):
@@ -68,38 +71,54 @@ def _parse_args(x):
     return DotDict(dict(zip(map(lambda x: x[2:], args.keys()), map(_parse_args, args.values()))))
 
 
+def get_structure_name(object):
+    file_name = object['Key'].split("/")[-1]
+    structure_name = file_name.split(".")[0] + "_structure.json"
+    return structure_name
+
+
 def scan(config):
     print("scanning")
     s3 = boto3.client("s3")
-    all_objects = s3.list_objects_v2(Bucket=S3_BUCKET, Prefix=config.s3_prefix)
+    all_objects = s3.list_objects_v2(Bucket=S3_BUCKET_IN, Prefix=config.s3_prefix)
 
     objects = [obj for obj in all_objects['Contents']
                if obj['Size'] > 1000]
 
-    # db_objects = dim_run['file_path_data'].tolist()
-    # print(db_objects)
-    # print(len([obj for obj in objects if obj['Key'] not in db_objects]))
-    # objects = [obj for obj in objects if obj['Key'] not in db_objects]
     objects = [obj for obj in objects if "PredictionDiagnostics" in obj['Key']
                and "x" not in obj['Key']
                and "Complete" not in obj['Key']
                # and obj['LastModified'] < datetime.datetime(2020, 3, 24, 5, 35, 43, tzinfo=tzutc())
-               and "_000159_" in obj['Key']
+               # and "_000175_" in obj['Key']
                ]
+
+    old_objects = []
+    old = datetime.datetime.now(pytz.utc) - datetime.timedelta(hours=6)
+    for obj in objects:
+        name = config.s3_output + '/' + get_structure_name(obj)
+        structure_objects = s3.list_objects_v2(Bucket=S3_BUCKET_OUT, Prefix=name)
+        # print(structure_objects)
+        if 'Contents' in structure_objects.keys() and len(structure_objects['Contents']) == 1:
+            if structure_objects['Contents'][0]['LastModified'] < old:
+                old_objects.append(obj)
+        else:
+            old_objects.append(obj)
+
+    objects = old_objects
     print(len(objects))
 
     events = KinesisEvents(service='S3Syncer', mode=config.mode)
     objects.reverse()
     for obj in objects:
         retrigger_data = {
             "filename": obj['Key'],
-            "bucket": S3_BUCKET,
+            "bucket": S3_BUCKET_IN,
             "size": obj['Size'],
             "hash": obj["ETag"].strip('\"')
         }
         events.put_upload_retrigger_event('complete', retrigger_data)
         print(retrigger_data)
-        time.sleep(1)
+        time.sleep(0.1)
 
 
 if __name__ == "__main__":

setup.py

@@ -18,7 +18,7 @@
 
 setup(name="beep",
       url="https://github.com/TRI-AMDD/beep",
-      version="2020.4.9",
+      version="2020.4.10",
       description=DESCRIPTION,
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',