Pass instance data to SNS topic

- Pass the instance data from `describe_instances` to the SNS topic

- If we get instance data on the SNS topic, use it instead of looking it all up again

- Sanitize instance data; it may contain un-JSON-serializable values like datetime(s)

RE: #5

ebs_snapper/lambdas.py

@@ -85,11 +85,15 @@ def lambda_snapshot(event, context):
             LOG.warn('lambda_snapshot missing specific keys: %s', str(event))
             continue
 
+        if 'instance_data' in message_json:
+            opt_instance_data = message_json['instance_data']
+
         # call the snapshot perform method
         snapshot.perform_snapshot(
             message_json['region'],
             message_json['instance_id'],
             message_json['settings'],
+            instance_data=opt_instance_data,
             context=context)
 
         LOG.info('Function lambda_snapshot completed')

ebs_snapper/shell.py

@@ -149,11 +149,15 @@ def shell_snapshot(*args):
     """Check for snapshots, executing if needed, like lambda version."""
     message_json = json.loads(args[0].message)
 
+    if 'instance_data' in message_json:
+        d = message_json['instance_data']
+
     # call the snapshot perform method
     snapshot.perform_snapshot(
         message_json['region'],
         message_json['instance_id'],
-        message_json['settings'])
+        message_json['settings'],
+        instance_data=d)
 
     LOG.info('Function shell_snapshot completed')
 

ebs_snapper/snapshot.py

@@ -96,29 +96,37 @@ def send_message_instances(region, sns_topic, configuration_snapshot, filters):
                 instance_id=instance['InstanceId'],
                 region=region,
                 topic_arn=sns_topic,
-                snapshot_settings=configuration_snapshot)
+                snapshot_settings=configuration_snapshot,
+                instance_data=instance)
 
 
-def send_fanout_message(instance_id, region, topic_arn, snapshot_settings):
+def send_fanout_message(instance_id, region, topic_arn, snapshot_settings, instance_data=None):
     """Publish an SNS message to topic_arn that specifies an instance and region to review"""
-    message = json.dumps({'instance_id': instance_id,
-                          'region': region,
-                          'settings': snapshot_settings})
+    data_hash = {'instance_id': instance_id,
+                 'region': region,
+                 'settings': snapshot_settings}
+
+    if instance_data:
+        data_hash['instance_data'] = sanitize_serializable(instance_data)
+
+    message = json.dumps(data_hash)
 
     LOG.info('send_fanout_message: %s', message)
 
     utils.sns_publish(TopicArn=topic_arn, Message=message)
 
 
-def perform_snapshot(region, instance, snapshot_settings, context=None):
+def perform_snapshot(region, instance, snapshot_settings, instance_data=None, context=None):
     """Check the region and instance, and see if we should take any snapshots"""
     LOG.info('Reviewing snapshots in region %s on instance %s', region, instance)
 
     # parse out snapshot settings
     retention, frequency = utils.parse_snapshot_settings(snapshot_settings)
 
-    # grab the data about this instance id
-    instance_data = utils.get_instance(instance, region)
+    # grab the data about this instance id, if we don't already have it
+    if instance_data is None or 'BlockDeviceMappings' not in instance_data:
+        instance_data = utils.get_instance(instance, region)
+
     ami_id = instance_data['ImageId']
 
     for dev in instance_data.get('BlockDeviceMappings', []):
@@ -180,3 +188,28 @@ def should_perform_snapshot(frequency, now, volume_id, recent=None):
         return expected_next < now
 
     raise Exception('Could not determine if snapshot was due', frequency, recent)
+
+
+def sanitize_serializable(instance_data):
+    """Check every value is serializable, build new dict with safe values"""
+    output = {}
+
+    # we can't serialize all values, so just grab the ones we can
+    for k, v in instance_data.iteritems():
+        can_ser = can_serialize_json(k, v)
+        if not can_ser:
+            continue
+
+        output[k] = v
+
+    return output
+
+
+def can_serialize_json(key, value):
+    """Return true if it's safe to pass this to json.dumps()"""
+
+    try:
+        json.dumps({key: value})
+        return True
+    except:
+        return False

tests/test_snapshot.py

@@ -90,18 +90,18 @@ def test_perform_fanout_by_region_snapshot(mocker):
     dummy_regions = ['us-west-2', 'us-east-1']
 
     # make some dummy instances in two regions
-    instance_maps = {}
+    region_map = {}
     for dummy_region in dummy_regions:
         client = boto3.client('ec2', region_name=dummy_region)
         create_results = client.run_instances(ImageId='ami-123abc', MinCount=5, MaxCount=5)
         for instance_data in create_results['Instances']:
-            instance_maps[instance_data['InstanceId']] = dummy_region
+            region_map[instance_data['InstanceId']] = dummy_region
 
     # need to filter instances, so need dynamodb present
     mocks.create_dynamodb('us-east-1')
     config_data = {
         "match": {
-            "instance-id": instance_maps.keys()
+            "instance-id": region_map.keys()
         },
         "snapshot": {
             "retention": "4 days",
@@ -117,12 +117,16 @@ def test_perform_fanout_by_region_snapshot(mocker):
     # fan out, and be sure we touched every instance we created before
     snapshot.perform_fanout_all_regions()
 
-    for key, value in instance_maps.iteritems():
+    print(snapshot.send_fanout_message.mock_calls)
+    for key, value in region_map.iteritems():
+        # refetch the instance data, to be sure it matches at call time
+        my_instance_data = utils.get_instance(key, value)
         snapshot.send_fanout_message.assert_any_call(  # pylint: disable=E1103
             instance_id=key,
             region=value,
             topic_arn=expected_sns_topic,
-            snapshot_settings=config_data)
+            snapshot_settings=config_data,
+            instance_data=my_instance_data)
 
 
 @mock_ec2