use PubSub JWT auth for security instead of the hardcoded static key

Author: JoshuaFox

cron_empty.yaml

@@ -1,6 +1 @@
-# Documented in Google App Engine https://cloud.google.com/appengine/docs/standard/python3/scheduling-jobs-with-cron-yaml
-cron:
-  - description: "Invoke /schedule which sends out messages triggering do_label per project/plugin."
-    url: /schedule
-    schedule: every day 10:00
-    target: iris3
+cron:

deploy.sh

@@ -59,8 +59,12 @@ while getopts 'cepoh' opt; do
             to do both; this is equivalent to -p -o
             Flags:
                   -o: Deploy org-level elements like Log Sink
-                  -p: Deploy project-level elements. Org-level elements are still required.
-                  The default if neither -o or -p are given is to enable both.
+                  -p: Deploy project-level elements.
+                  Org-level elements are a pre-requisite.
+                  This is useful for redeploying to the same project, e.g., to change config.
+                  If you want to deploy to a *different* project,
+                  then you have to deploy the org-level elements.
+                  The default, if neither -o or -p are given, is to enable both.
                   -c: Label on Cloud Scheduler cron to add labels
                   -e: Label on-creation-event.
                   The default if neither -c or -e are given is to enable both.
@@ -104,7 +108,7 @@ gcloud projects describe "$PROJECT_ID" >/dev/null|| {
   exit 1
 }
 
-echo "Project ID $PROJECT_ID"
+#echo "Project ID $PROJECT_ID"
 gcloud config set project "$PROJECT_ID"
 
 

main.py

@@ -174,11 +174,12 @@ def __send_pubsub_per_projectplugin(configured_projects):
 @app.route("/label_one", methods=["POST"])
 def label_one():
     """Message received from PubSub when the log sink detects a new resource"""
+    logging.info("label_one called")
+    increment_invocation_count("label_one")
     ok = __check_pubsub_jwt()
     if not ok:
         return "JWT Failed", 400
 
-    increment_invocation_count("label_one")
     with gae_memory_logging("label_one"):
 
         plugins_found = []
@@ -234,6 +235,10 @@ def label_one():
 
 def __check_pubsub_jwt():
     try:
+        #TODO the sample https://github.com/GoogleCloudPlatform/python-docs-samples/blob/ff4c1d55bb5b6995c63383469535604002dc9ba2/appengine/standard_python3/pubsub/main.py#L69
+        # has this. I am not sure why or how the token arg gets there.
+        # if request.args.get("token", "") != current_app.config["PUBSUB_VERIFICATION_TOKEN"]:
+        #     return "Invalid request", 400
         bearer_token = flask.request.headers.get("Authorization")
         token = bearer_token.split(" ")[1]
 
@@ -319,11 +324,12 @@ def do_label():
     """Receives a push message from PubSub, sent from schedule() above,
     and labels all objects of a given plugin and project_id.
     """
-
+    increment_invocation_count("do_label")
+    logging.info("do_label called")
     ok = __check_pubsub_jwt()
     if not ok:
         return "JWT Failed", 400
-    increment_invocation_count("do_label")
+
     with gae_memory_logging("do_label"):
 
         project_id = ""  # set up variables to allow logging in Exception block at end

scripts/_deploy-org.sh

@@ -37,10 +37,11 @@ set -e
 
 if [[ "$role_error" != "0" ]]; then
   echo "Error in accessing organization.
-    Please either run ./deploy.sh -p to depoly only project-level elements
-   (generally what you need for an incremental upgrade)
-   or run it this script using a role that has organization permissions to
-   deploy organization-level elements like roles or Log Sink."
+   If you just want to redeploy to the same project,
+   e.g., to upgrade the config, and you have the necessary
+   project role but not the necessary org role,
+   please run ./deploy.sh -p .
+   Or get yourself the org-level role as documented in README."
   exit $role_error
 fi
 

scripts/_deploy-project.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 #
-# Deploys Iris to Google App Engine, setting up   Sinks, Topics, and Subscriptions as needed.
+# Deploys Iris to Google App Engine,
+# first setting up Sinks, Topics, Subscriptions, and Role Bindings as needed.
 # Usage
 # - Called from deploy.sh
 # - Pass the project as the first command line argument.
@@ -102,7 +103,13 @@ PUBSUB_SERVICE_ACCOUNT="service-${project_number}@gcp-sa-pubsub.iam.gserviceacco
 
 msg_sender_sa_name=iris-msg-sender
 
-gcloud iam service-accounts create --project $PROJECT_ID $msg_sender_sa_name ||true
+set +e
+gcloud iam service-accounts describe  ${msg_sender_sa_name}@${PROJECT_ID}.iam.gserviceaccount.com --project $PROJECT_ID
+if [[ $? -ne 0 ]]; then
+  set -e
+  gcloud iam service-accounts create --project $PROJECT_ID $msg_sender_sa_name
+fi
+set -e
 
 MSGSENDER_SERVICE_ACCOUNT=${msg_sender_sa_name}@${PROJECT_ID}.iam.gserviceaccount.com
 
@@ -129,7 +136,6 @@ if [[ $? -ne 0  && $BINDING_ERR_OUTPUT == *"gcp-sa-pubsub.iam.gserviceaccount.co
            --role="roles/pubsub.publisher" --project $PROJECT_ID >/dev/null
 
 fi
-
 set -e
 
 # Create PubSub subscription receiving commands from the /schedule handler that is triggered from cron
@@ -197,7 +203,8 @@ else
         --max-retry-delay=$MAX_RETRY \
         --quiet >/dev/null
   else
-      gcloud pubsub subscriptions create "$LABEL_ONE_SUBSCRIPTION" --topic "$LOGS_TOPIC" --project="$PROJECT_ID" \
+      gcloud pubsub subscriptions create "$LABEL_ONE_SUBSCRIPTION" \
+         --topic "$LOGS_TOPIC" --project="$PROJECT_ID" \
         --push-endpoint="$LABEL_ONE_SUBSCRIPTION_ENDPOINT" \
         --push-auth-service-account $MSGSENDER_SERVICE_ACCOUNT  \
         --ack-deadline=$ACK_DEADLINE \
@@ -214,6 +221,8 @@ else
       --role="roles/pubsub.subscriber" --project $PROJECT_ID >/dev/null
 fi
 
+
+
 if [[ "$LABEL_ON_CRON" == "true" ]]; then
     cp cron_full.yaml cron.yaml
 else

test_scripts/integration_test.py

@@ -208,7 +208,6 @@ def create_and_describe_resources(test_project, run_id, gce_zone):
     create_resources(test_project, run_id, gce_zone)
     # Next line necessary to let the labels be visible for PubSub topics and subscriptions.
     # Could speed it up by repeatedly checking for the labels
-    # up to 5 times, sleeping  2 sec in between each check.
     time.sleep(30)
     describe_resources(test_project, run_id, gce_zone)
 
@@ -295,7 +294,7 @@ def pause_for_user_input():
         print("Type ENTER to proceed to clean up resources ")
         _ = sys.stdin.readline()
 
-    # pause_for_user_input()# Use this in debugging, to keep the test resources alive until you hit E
+    #pause_for_user_input()# Use this in debugging, to keep the test resources alive until you hit E
 
     remove_config_file()
     commands = [

uninstall.sh

@@ -56,18 +56,22 @@ while getopts 'poh' opt; do
     cat <<EOF
       Usage uninstall.sh PROJECT_ID 
           Argument:
-                  The project to which Iris was  deployed
+            The project to which Iris was  deployed
           Options, to be given before project ID.
             If neither -p nor -o is given, the default behavior is used:
-            Both are uninstalled;  equivalent to -p -o
+            both are uninstalled; this is equivalent to  giving both -p -o
             Flags:
                   -p: Uninstall project-level elements of Iris.
                   This is useful if you deployed Iris to two projects
                   in an org and want to delete it on one of those.
+                  Warning: If you delete the project-level elements,
+                  and the log sink still exists in the org, you will get
+                  error-message emails from Google (since the log sink
+                  no longer has a topic to send to)
                   -o: Uninstall org-level elements like Log Sink
             Environment variable:
                   IRIS_CUSTOM_ROLE (Optional, default is iris3) An identifier for
-                  the Iris custom role that you want to delete.
+                  the Iris custom role that was used, if non-default.
 EOF
     exit 1
     ;;
@@ -95,7 +99,7 @@ gcloud projects describe "$PROJECT_ID" >/dev/null`` || {
   exit 1
 }
 
-echo "Project ID $PROJECT_ID"
+#echo "Project ID $PROJECT_ID"
 gcloud config set project "$PROJECT_ID"
 
 

uninstall_scripts/_uninstall-for-org.sh

@@ -22,7 +22,8 @@ gcloud organizations remove-iam-policy-binding "$ORGID" --all \
   --member "serviceAccount:$PROJECT_ID@appspot.gserviceaccount.com" \
   --role "organizations/$ORGID/roles/$IRIS_CUSTOM_ROLE" >/dev/null|| true
 
-gcloud iam roles delete -q "$IRIS_CUSTOM_ROLE" --organization "$ORGID" >/dev/null || true
+# Just leave the role; it causes too much trouble in its "soft delete" state
+#gcloud iam roles delete -q "$IRIS_CUSTOM_ROLE" --organization "$ORGID" >/dev/null || true
 
 if  gcloud logging sinks describe --organization="$ORGID" "$LOG_SINK" >&/dev/null; then
     svcaccount=$(gcloud logging sinks describe --organization="$ORGID" "$LOG_SINK" |

uninstall_scripts/_uninstall-for-project.sh

@@ -28,13 +28,12 @@ gcloud pubsub subscriptions remove-iam-policy-binding $DO_LABEL_SUBSCRIPTION \
     --role="roles/pubsub.subscriber" --project $PROJECT_ID || true
 
 gcloud pubsub subscriptions remove-iam-policy-binding $LABEL_ONE_SUBSCRIPTION \
-      --member="serviceAccount:$PUBSUB_SERVICE_ACCOUNT"\
-      --role="roles/pubsub.subscriber" --project $PROJECT_ID ||true
-
-gcloud projects remove-iam-policy-binding --project ${PROJECT_ID} \
- --member="serviceAccount:${MSGSENDER_SERVICE_ACCOUNT}"\
- --role='roles/iam.serviceAccountTokenCreator'
+    --member="serviceAccount:$PUBSUB_SERVICE_ACCOUNT"\
+    --role="roles/pubsub.subscriber" --project $PROJECT_ID ||true
 
+gcloud projects remove-iam-policy-binding ${PROJECT_ID}  \
+    --member="serviceAccount:${MSGSENDER_SERVICE_ACCOUNT}"\
+    --role='roles/iam.serviceAccountTokenCreator' ||true
 
 gcloud pubsub subscriptions delete $DEADLETTER_SUB --project="$PROJECT_ID" -q || true
 gcloud pubsub subscriptions delete "$DO_LABEL_SUBSCRIPTION" -q --project="$PROJECT_ID" ||true