use PubSub JWT auth for security instead of the hardcoded static key

Author: JoshuaFox

.gitignore

@@ -1,13 +1,18 @@
 
-# See config.yaml.original in git; the others are dynamic and so not put in git.
+# See config.yaml.original and cron_.*.yaml in git;
+# the below are dynamic and so not put in git.
 config-test.yaml
 config-dev.yaml
 config.yaml
 cron.yaml
+#
+
 #
 __pycache__/
 *.bak
 .DS_Store
+*.swp
+
 TEMP*
 .idea/
 

iris-custom-role.yaml

@@ -3,7 +3,7 @@ description: >
   Permissions needed by the custom role used by the Iris App Engine app's service account.
   A nondefault name for this role can be passed as env variable IRIS_CUSTOM_ROLE 
   into the deployment and uninstall scripts.
-# TODO For DRY, embed the following values in the Python class, then call these classes to construct this.
+# TODO Each Python plugin class should expose these and we should pull it from there,to have the info in one place.
 #
 # TODO: Not clear that `setTags` is really needed.
 stage: "GA"

main.py

@@ -235,7 +235,7 @@ def label_one():
 
 def __check_pubsub_jwt():
     try:
-        #TODO the sample https://github.com/GoogleCloudPlatform/python-docs-samples/blob/ff4c1d55bb5b6995c63383469535604002dc9ba2/appengine/standard_python3/pubsub/main.py#L69
+        # TODO the sample https://github.com/GoogleCloudPlatform/python-docs-samples/blob/ff4c1d55bb5b6995c63383469535604002dc9ba2/appengine/standard_python3/pubsub/main.py#L69
         # has this. I am not sure why or how the token arg gets there.
         # if request.args.get("token", "") != current_app.config["PUBSUB_VERIFICATION_TOKEN"]:
         #     return "Invalid request", 400

plugins/bigquery.py

@@ -23,7 +23,6 @@ def _discovery_api():
     @lru_cache(maxsize=500)  # cached per project
     def _cloudclient(cls, project_id=None):
         assert project_id, "'None' is only for the signature"
-        logging.info("_cloudclient for %s", cls.__name__)
         # Local import to avoid burdening AppEngine memory.
         # Loading all Cloud Client libraries would be 100MB  means that
         # the default AppEngine Instance crashes on out-of-memory even before actually serving a request.

plugins/buckets.py

@@ -20,7 +20,6 @@ def method_names():
     @lru_cache(maxsize=500)  # cached per project
     def _cloudclient(cls, project_id=None):
         assert project_id, "'None' is only for the signature"
-        logging.info("_cloudclient for %s", cls.__name__)
         # Local import to avoid burdening AppEngine memory.
         # Loading all Cloud Client libraries would be 100MB  means that
         # the default AppEngine Instance crashes on out-of-memory even before actually serving a request.

plugins/cloudsql.py

@@ -18,7 +18,6 @@ def method_names():
 
     @classmethod
     def _cloudclient(cls, _=None):
-        logging.info("_cloudclient for %s", cls.__name__)
         raise NotImplementedError(
             "There is no Cloud Client library for " + cls.__name__
         )

plugins/disks.py

@@ -22,7 +22,6 @@ class Disks(GceZonalBase):
     @staticmethod
     @lru_cache(maxsize=1)
     def _create_cloudclient():
-        logging.info("_cloudclient for %s", "Disks")
         # Local import to avoid burdening AppEngine memory.
         # Loading all Cloud Client libraries would be 100MB  means that
         # the default AppEngine Instance crashes on out-of-memory even before actually serving a request.

plugins/instances.py

@@ -17,8 +17,6 @@ class Instances(GceZonalBase):
     @staticmethod
     @lru_cache(maxsize=1)
     def _create_cloudclient():
-
-        logging.info("_cloudclient for %s", "Instances")
         # Local import to avoid burdening AppEngine memory.
         # Loading all Cloud Client libraries would be 100MB  means that
         # the default AppEngine Instance crashes on out-of-memory even before actually serving a request.

plugins/snapshots.py

@@ -13,8 +13,6 @@ class Snapshots(GceBase):
     @classmethod
     @lru_cache(maxsize=1)
     def _cloudclient(cls, _=None):
-
-        logging.info("_cloudclient for %s", cls.__name__)
         # Local import to avoid burdening AppEngine memory. Loading all
         # Client libraries would be 100MB  means that the default AppEngine
         # Instance crashes on out-of-memory even before actually serving a request.

plugins/subscriptions.py

@@ -17,7 +17,6 @@ class Subscriptions(Plugin):
     @classmethod
     @lru_cache(maxsize=1)
     def _cloudclient(cls, _=None):
-        logging.info("_cloudclient for %s", cls.__name__)
         # Local import to avoid burdening AppEngine memory.
         # Loading all Cloud Client libraries would be 100MB  means that
         # the default AppEngine Instance crashes on out-of-memory even before actually serving a request.

plugins/topics.py

@@ -17,7 +17,7 @@ class Topics(Plugin):
     @classmethod
     @lru_cache(maxsize=1)
     def _cloudclient(cls, _=None):
-        logging.info("_cloudclient for %s", cls.__name__)
+
         # Local import to avoid burdening AppEngine memory.
         # Loading all Cloud Client libraries would be 100MB  means that
         # the default AppEngine Instance crashes on out-of-memory even before actually serving a request.

scripts/_deploy-org.sh

@@ -79,7 +79,7 @@ else
   fi
 
   # Add methodName filter to the log sink
-  #TODO get this directly from the Python class to avoid duplication
+  #TODO Each Python plugin class should expose these and we should pull it from there,to have the info in one place.
   log_filter+=('protoPayload.methodName:(')
   log_filter+=('"storage.buckets.create"')
   log_filter+=('OR "compute.instances.insert" OR "compute.instances.start" OR "datasetservice.insert"')

scripts/_deploy-project.sh

@@ -114,7 +114,7 @@ set -e
 MSGSENDER_SERVICE_ACCOUNT=${msg_sender_sa_name}@${PROJECT_ID}.iam.gserviceaccount.com
 
 gcloud projects add-iam-policy-binding ${PROJECT_ID} \
- --member="serviceAccount:${MSGSENDER_SERVICE_ACCOUNT}"\
+ --member="serviceAccount:${PUBSUB_SERVICE_ACCOUNT}"\
  --role='roles/iam.serviceAccountTokenCreator'
 
 set +e
@@ -173,10 +173,9 @@ fi
 
 # Allow Pubsub to delete failed message from this sub
 gcloud pubsub subscriptions add-iam-policy-binding $DO_LABEL_SUBSCRIPTION \
-    --member="serviceAccount:$PUBSUB_SERVICE_ACCOUNT"\
+    --member="serviceAccount:$PUBSUB_SERVICE_ACCOUNT" \
     --role="roles/pubsub.subscriber" --project $PROJECT_ID >/dev/null
 
-
 if [[ "$LABEL_ON_CREATION_EVENT" != "true" ]]; then
   echo >&2 "Will not label on creation event."
   gcloud pubsub subscriptions delete "$LABEL_ONE_SUBSCRIPTION" --project="$PROJECT_ID" 2>/dev/null || true

test_scripts/integration_test.py

@@ -294,7 +294,7 @@ def pause_for_user_input():
         print("Type ENTER to proceed to clean up resources ")
         _ = sys.stdin.readline()
 
-    #pause_for_user_input()# Use this in debugging, to keep the test resources alive until you hit E
+    # pause_for_user_input()# Use this in debugging, to keep the test resources alive until you hit E
 
     remove_config_file()
     commands = [

uninstall_scripts/_uninstall-for-project.sh

@@ -31,9 +31,12 @@ gcloud pubsub subscriptions remove-iam-policy-binding $LABEL_ONE_SUBSCRIPTION \
     --member="serviceAccount:$PUBSUB_SERVICE_ACCOUNT"\
     --role="roles/pubsub.subscriber" --project $PROJECT_ID ||true
 
-gcloud projects remove-iam-policy-binding ${PROJECT_ID}  \
-    --member="serviceAccount:${MSGSENDER_SERVICE_ACCOUNT}"\
-    --role='roles/iam.serviceAccountTokenCreator' ||true
+# We don't do the following to avoid disrupting PubSub more generally in the proect.
+#gcloud projects remove-iam-policy-binding ${PROJECT_ID}  \
+#    --member="serviceAccount:${PUBSUB_SERVICE_ACCOUNT}"\
+#    --role='roles/iam.serviceAccountTokenCreator' ||true
+
+
 
 gcloud pubsub subscriptions delete $DEADLETTER_SUB --project="$PROJECT_ID" -q || true
 gcloud pubsub subscriptions delete "$DO_LABEL_SUBSCRIPTION" -q --project="$PROJECT_ID" ||true

util/utils.py

@@ -252,7 +252,7 @@ def run_command(command_s, extra_env: typing.Dict[str, str] = None):
     command = command_s.split(" ")
     env = os.environ
     if extra_env is not None:
-        env = env | extra_env
+        env |= extra_env
     result = subprocess.run(command, stdout=subprocess.PIPE, check=True, env=env)
     output = result.stdout.decode("utf-8")
     return output.strip("\n")