debug logging

Author: JoshuaFox

.gitignore

@@ -1,5 +1,6 @@
 # Byte-compiled / optimized / DLL files
 #
+.venv/**
 ./idea
 .idea/
 __pycache__/
@@ -109,3 +110,6 @@ venv.bak/
 /config.json
 config.json
 /dev_config.json
+**.bak
+# Datastore not used, but this is autogenerated
+index.yaml

appengine_config.py

@@ -1,5 +1,10 @@
 from google.appengine.ext import vendor
 import sys
+import os
 # Add any libraries installed in the "lib" folder.
+print("will add lib")
 vendor.add('lib')
+print('added lib')
+vendor.add(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib'))
+print('added abs path to lib')
 sys.path = ['lib'] + sys.path

deploy.sh

@@ -10,13 +10,14 @@ if [[ $# -eq 0 ]] ; then
  echo Missing project id argument
  exit
 fi
-
-PROJECTID=$(gcloud projects list | grep -iw "$1" | awk '{print $1}')
+echo xxxx $1 xxxx
+PROJECTID=$(gcloud projects list | grep -i "^$1 " | awk '{print $1}')
 
 if [ -z "$PROJECTID" ]; then
 	echo "Project $1 Not Found!"
 	exit 1
 fi
+echo "PROJECTID $PROJECTID XXX"
 
 #set the project context
 echo "Project ID $PROJECTID"

main.py

@@ -15,11 +15,12 @@
 app = Flask(__name__)
 plugins = []
 
-
 def store(key, value, chunksize=950000):
     serialized = cloudpickle.dumps(value, 2)
     values = {}
     for i in xrange(0, len(serialized), chunksize):
+        if chunksize>10**6:
+         print('setting memcache', key,'with chunk of size', chunksize, serialized[i:i + 100] )
         values['%s.%s' % (key, i // chunksize)] = serialized[i:i + chunksize]
     return memcache.set_multi(values)
 
@@ -121,9 +122,12 @@ def schedule():
 
 @app.route('/tasks/do_tag', methods=['GET'])
 def do_tag():
+    logging.info("do_tag")
     f = retrieve(request.args['plugin'])
+    logging.info("do_tag",f )
     if f is not None:
         project_id = request.args['project_id']
+        logging.info('do_tag', project_id)
         f.do_tag(project_id)
     return 'ok', 200
 

plugins/bigquery.py

@@ -1,5 +1,6 @@
 """ Taging BQ tabels and datasets."""
 import logging
+import traceback
 
 from google.auth import app_engine
 from googleapiclient import discovery, errors
@@ -113,6 +114,7 @@ def get_gcp_object(self, data):
             return table
         except Exception as e:
             logging.error(e)
+            print(traceback.format_exc())
 
         return None
 

plugins/gce.py

@@ -144,6 +144,7 @@ def get_instance(self, project_id, zone, name):
 
 
     def do_tag(self, project_id):
+        logging.info("do_tag GCE", project_id)
         for zone in self.get_zones(project_id):
             instances = self.list_instances(project_id, zone)
             for instance in instances:
@@ -169,15 +170,18 @@ def get_gcp_object(self, data):
 
 
     def tag_one(self, gcp_object, project_id):
+        logging.info("do_tag GCE", gcp_object)
         try:
             org_labels = {}
             org_labels = gcp_object['labels']
         except KeyError:
             pass
+        logging.info('labelFingerprint'+ str(gcp_object.get('labelFingerprint', '')))
         labels = dict(
             [('labelFingerprint', gcp_object.get('labelFingerprint', ''))])
         labels['labels'] = self.gen_labels(gcp_object)
         for k, v in org_labels.items():
+            logging.info('keyvalue'+k+str(v))
             labels['labels'][k] = v
         try:
             zone = gcp_object['zone']

utils/utils.py

@@ -64,6 +64,10 @@ def fatal_code(e):
 
 
 def is_service_enbaled(service_list, service):
+    if service_list is None:
+        print('service_list is None, service is ', service)
+        return False
+
     if 'services' in service_list:
         for srv in service_list['services']:
             if srv['config']['name'].lower() == service.lower():