warnings when integ test is run on a new project

Author: JoshuaFox

.gcloudignore

@@ -1,8 +1,9 @@
 
 test_scripts/
-
+uninstall_scripts/
+scripts/
 sample_data/
-memray/
+
 README.md
 *.sh
 TEMP*

deploy.sh

@@ -105,14 +105,20 @@ if [[ "$deploy_org" != "true" ]] && [[ "$deploy_proj" != "true" ]]; then
   deploy_proj=true
 fi
 
+gcloud services enable cloudresourcemanager.googleapis.com --project $PROJECT_ID > /dev/null ||true
+
 gcloud projects describe "$PROJECT_ID" >/dev/null || {
   echo "Project $PROJECT_ID not found"
   exit 1
 }
 
-gcloud auth application-default set-quota-project $PROJECT_ID > /dev/null 2>&1
-gcloud config set project "$PROJECT_ID" > /dev/null  2>&1
+gcloud config set project "$PROJECT_ID" > /dev/null
+
+export LOG_SINK=iris_log
 
+# Get organization id for this project
+ORGID=$(gcloud projects get-ancestors "$PROJECT_ID" --format='value(TYPE,ID)' | awk '/org/ {print $2}')
+export ORGID
 
 if [[ "$deploy_org" == "true" ]]; then
   ./scripts/_deploy-org.sh || exit 1

scripts/_deploy-org.sh

@@ -7,14 +7,9 @@
 #set -x
 # The following lines must come before set -u
 if [[ -z "$IRIS_CUSTOM_ROLE" ]]; then IRIS_CUSTOM_ROLE=iris3; fi
-if [[ -z "$SKIP_ADDING_IAM_BINDINGS" ]]; then SKIP_ADDING_IAM_BINDINGS=""; fi
 set -u
 set -e
 
-LOG_SINK=iris_log
-
-# Get organization id for this project
-ORGID=$(gcloud projects get-ancestors "$PROJECT_ID" --format='value(TYPE,ID)' | awk '/org/ {print $2}')
 
 set +e
 # Create custom role to run iris
@@ -50,13 +45,12 @@ fi
 gcloud organizations add-iam-policy-binding "$ORGID" \
   --member "serviceAccount:$PROJECT_ID@appspot.gserviceaccount.com" \
   --role "organizations/$ORGID/roles/$IRIS_CUSTOM_ROLE" \
-  --condition=None >/dev/null 2>&1
+  --condition=None >/dev/null
 
 if [[ "$LABEL_ON_CREATION_EVENT" != "true" ]]; then
   echo >&2 "Will not label on creation event."
   gcloud logging sinks delete -q --organization="$ORGID" "$LOG_SINK" || true
 else
-  # Create PubSub topic for receiving logs about new GCP objects
 
   log_filter=("")
 
@@ -91,26 +85,17 @@ else
   log_filter+=(')')
 
   # Create or update a sink at org level
+  # Logs topic does not yet exist!
   if ! gcloud logging sinks describe --organization="$ORGID" "$LOG_SINK" >&/dev/null; then
     gcloud logging sinks create "$LOG_SINK" \
       pubsub.googleapis.com/projects/"$PROJECT_ID"/topics/"$LOGS_TOPIC" \
       --organization="$ORGID" --include-children \
-      --log-filter="${log_filter[*]}" --quiet >/dev/null 2>&1
+      --log-filter="${log_filter[*]}" --quiet >/dev/null
   else
     gcloud logging sinks update "$LOG_SINK" \
       pubsub.googleapis.com/projects/"$PROJECT_ID"/topics/"$LOGS_TOPIC" \
       --organization="$ORGID" \
-      --log-filter="${log_filter[*]}" --quiet >/dev/null 2>&1
+      --log-filter="${log_filter[*]}" --quiet >/dev/null
   fi
 
-  # Extract service account from sink configuration.
-  # This is the service account that publishes to PubSub.
-  svcaccount=$(gcloud logging sinks describe --organization="$ORGID" "$LOG_SINK" |
-    grep writerIdentity | awk '{print $2}')
-
-  if [[ "$SKIP_ADDING_IAM_BINDINGS" != "true" ]]; then
-    # Assign a publisher role to the extracted service account.
-    gcloud projects add-iam-policy-binding "$PROJECT_ID" \
-      --member="$svcaccount" --role=roles/pubsub.publisher --quiet >/dev/null 2>&1
-  fi
 fi

scripts/_deploy-project.sh

@@ -36,16 +36,18 @@ fi
 appengineHostname=$(gcloud app describe --project $PROJECT_ID | grep defaultHostname | cut -d":" -f2 | awk '{$1=$1};1')
 if [[ -z "$appengineHostname" ]]; then
   echo "App Engine is not enabled in $PROJECT_ID.
-   To do this, please enable it with \"gcloud app create [--region=REGION]\",
-   and then deploy a simple \"Hello World\" default service to enable App Engine."
+  As a pre-req for this, as for all App Engine services, please
+  (1) Create the app with  \"gcloud app create [--region=REGION]\",
+  (2) and then deploy a simple \"Hello World\" default-service."
   exit 1
 fi
 
 appengine_sa_has_editor_role=$(gcloud projects get-iam-policy ${PROJECT_ID} \
   --flatten="bindings[].members" \
-  --format='table(bindings.role)' \
+  --format='table[no-heading](bindings.role)' \
   --filter="bindings.members:${PROJECT_ID}@appspot.gserviceaccount.com" | grep "roles/editor" || true)
 
+
 if [ -z "$appengine_sa_has_editor_role" ]; then
   echo "Must bind role Project Editor for project ${PROJECT_ID} to service account ${PROJECT_ID}@appspot.gserviceaccount.com.
       (The binding exists by default but is missing.)"
@@ -79,10 +81,35 @@ required_svcs=(
 )
 for svc in "${required_svcs[@]}"; do
   if ! [ ${enabled_services["$svc"]+_} ]; then
-    gcloud services enable "$svc"
+    gcloud services enable "$svc" --project $PROJECT_ID
   fi
 done
 
+
+# Extract service account from sink configuration.
+# This is the service account that publishes to PubSub.
+sink_svc_account=$(gcloud logging sinks describe --organization="$ORGID" "$LOG_SINK" |
+  grep writerIdentity | awk '{print $2}')
+
+if [[ "$SKIP_ADDING_IAM_BINDINGS" != "true" ]]; then
+  # Assign a publisher role to the extracted service account.
+  gcloud projects add-iam-policy-binding "$PROJECT_ID" \
+    --member="$sink_svc_account" --role=roles/pubsub.publisher --quiet >/dev/null
+else
+  #Check to avoid situation where a test does SKIP_ADDING_IAM_BINDINGS and they don't at all exist
+  already_have_binding=$(  gcloud projects get-iam-policy $PROJECT_ID  \
+    --flatten="bindings[].members" \
+    --format='table[no-heading](bindings.role)' \
+    --filter="bindings.members:${sink_svc_account}" |grep "pubsub.publisher" ||true)
+
+  if [ -z $already_have_binding ]; then
+    echo "SKIP_ADDING_IAM_BINDINGS is meant for tests
+    (to avoid overloading the quotas when you repeated run the tests).
+    First deploy the usual way before running the tests."
+    exit 1
+  fi
+fi
+
 # Create PubSub topic for receiving commands from the /schedule handler that is triggered from cron
 gcloud pubsub topics describe "$SCHEDULELABELING_TOPIC" --project="$PROJECT_ID" &>/dev/null ||
   gcloud pubsub topics create "$SCHEDULELABELING_TOPIC" --project="$PROJECT_ID" --quiet >/dev/null
@@ -99,21 +126,21 @@ if gcloud pubsub subscriptions describe "$DEADLETTER_SUB" --project="$PROJECT_ID
     gcloud pubsub subscriptions update $DEADLETTER_SUB \
     --project="$PROJECT_ID" \
     --message-retention-duration=2d \
-    --quiet >/dev/null 2>&1
+    --quiet >/dev/null
 
 else
    gcloud pubsub subscriptions create $DEADLETTER_SUB \
     --project="$PROJECT_ID" \
     --topic $DEADLETTER_TOPIC \
     --message-retention-duration=2d \
-    --quiet >/dev/null 2>&1
+    --quiet >/dev/null
 fi
 
 project_number=$(gcloud projects describe $PROJECT_ID --format json | jq -r '.projectNumber')
 PUBSUB_SERVICE_ACCOUNT="service-${project_number}@gcp-sa-pubsub.iam.gserviceaccount.com"
 # The following line is only needed on first deployment, and so slows things
 # down unnecessarily otherwise. But most users do not install Iris repeatedly.
-gcloud beta services identity create --project $PROJECT_ID --service pubsub >/dev/null 2>&1
+gcloud beta services identity create --project $PROJECT_ID --service pubsub >/dev/null
 
 if ! gcloud iam service-accounts describe iris-msg-sender@${PROJECT_ID}.iam.gserviceaccount.com --project $PROJECT_ID >/dev/null ;
 then
@@ -139,7 +166,7 @@ then
     --dead-letter-topic=$DEADLETTER_TOPIC \
     --min-retry-delay=$MIN_RETRY \
     --max-retry-delay=$MAX_RETRY \
-    --quiet >/dev/null 2>&1
+    --quiet >/dev/null
 else
   gcloud pubsub subscriptions create "$DO_LABEL_SUBSCRIPTION" \
     --topic "$SCHEDULELABELING_TOPIC" --project="$PROJECT_ID" \
@@ -150,7 +177,7 @@ else
     --dead-letter-topic=$DEADLETTER_TOPIC \
     --min-retry-delay=$MIN_RETRY \
     --max-retry-delay=$MAX_RETRY \
-    --quiet >/dev/null 2>&1
+    --quiet >/dev/null
 fi
 
 if [[ "$LABEL_ON_CREATION_EVENT" != "true" ]];
@@ -160,7 +187,7 @@ then
 else
   # Create PubSub topic for receiving logs about new GCP objects
   gcloud pubsub topics describe "$LOGS_TOPIC" --project="$PROJECT_ID" &>/dev/null ||
-    gcloud pubsub topics create $LOGS_TOPIC --project="$PROJECT_ID" --quiet >/dev/null 2>&1
+    gcloud pubsub topics create $LOGS_TOPIC --project="$PROJECT_ID" --quiet >/dev/null
 
   # Create or update PubSub subscription for receiving log about new GCP objects
   if gcloud pubsub subscriptions describe "$LABEL_ONE_SUBSCRIPTION" --project="$PROJECT_ID" &>/dev/null ;
@@ -173,7 +200,7 @@ else
       --dead-letter-topic=$DEADLETTER_TOPIC \
       --min-retry-delay=$MIN_RETRY \
       --max-retry-delay=$MAX_RETRY \
-      --quiet >/dev/null 2>&1
+      --quiet >/dev/null
   else
     gcloud pubsub subscriptions create "$LABEL_ONE_SUBSCRIPTION" \
       --topic "$LOGS_TOPIC" --project="$PROJECT_ID" \
@@ -184,15 +211,14 @@ else
       --dead-letter-topic=$DEADLETTER_TOPIC \
       --min-retry-delay=$MIN_RETRY \
       --max-retry-delay=$MAX_RETRY \
-      --quiet >/dev/null 2>&1
+      --quiet >/dev/null
   fi
 
 fi
 
 gcloud pubsub topics describe "$LABEL_ALL_TOPIC" --project="$PROJECT_ID" &>/dev/null ||
   gcloud pubsub topics create $LABEL_ALL_TOPIC --project="$PROJECT_ID" --quiet >/dev/null
 
-
 if gcloud pubsub subscriptions describe "$LABEL_ALL_SUBSCRIPTION" --project="$PROJECT_ID" &>/dev/null; then
   gcloud pubsub subscriptions update "$LABEL_ALL_SUBSCRIPTION" \
     --project="$PROJECT_ID" \
@@ -203,7 +229,7 @@ if gcloud pubsub subscriptions describe "$LABEL_ALL_SUBSCRIPTION" --project="$PR
     --dead-letter-topic=$DEADLETTER_TOPIC \
     --min-retry-delay=$MIN_RETRY \
     --max-retry-delay=$MAX_RETRY \
-    --quiet >/dev/null 2>&1
+    --quiet >/dev/null
 else
   gcloud pubsub subscriptions create "$LABEL_ALL_SUBSCRIPTION" \
     --topic "$LABEL_ALL_TOPIC" --project="$PROJECT_ID" \
@@ -221,28 +247,28 @@ if [[ "$LABEL_ON_CREATION_EVENT" == "true" ]]; then
   # Allow Pubsub to delete failed message from this sub
   gcloud pubsub subscriptions add-iam-policy-binding $DO_LABEL_SUBSCRIPTION \
     --member="serviceAccount:$PUBSUB_SERVICE_ACCOUNT" \
-    --role="roles/pubsub.subscriber" --project $PROJECT_ID >/dev/null 2>&1
+    --role="roles/pubsub.subscriber" --project $PROJECT_ID >/dev/null
 
 fi
 
 gcloud pubsub subscriptions add-iam-policy-binding $LABEL_ALL_SUBSCRIPTION \
   --member="serviceAccount:$PUBSUB_SERVICE_ACCOUNT" \
-  --role="roles/pubsub.subscriber" --project $PROJECT_ID >/dev/null 2>&1
+  --role="roles/pubsub.subscriber" --project $PROJECT_ID >/dev/null
 
 # Allow Pubsub to delete failed message from this sub
 gcloud pubsub subscriptions add-iam-policy-binding $LABEL_ONE_SUBSCRIPTION \
   --member="serviceAccount:$PUBSUB_SERVICE_ACCOUNT" \
-  --role="roles/pubsub.subscriber" --project $PROJECT_ID >/dev/null 2>&1
+  --role="roles/pubsub.subscriber" --project $PROJECT_ID >/dev/null
 
 # Allow Pubsub to publish into the deadletter topic
 gcloud pubsub topics add-iam-policy-binding $DEADLETTER_TOPIC \
   --member="serviceAccount:$PUBSUB_SERVICE_ACCOUNT" \
-  --role="roles/pubsub.publisher" --project "$PROJECT_ID" >/dev/null 2>&1
+  --role="roles/pubsub.publisher" --project "$PROJECT_ID" >/dev/null
 
 if [[ "$SKIP_ADDING_IAM_BINDINGS" != "true" ]]; then
   gcloud projects add-iam-policy-binding "${PROJECT_ID}" \
     --member="serviceAccount:${PUBSUB_SERVICE_ACCOUNT}" \
-    --role='roles/iam.serviceAccountTokenCreator' >/dev/null 2>&1
+    --role='roles/iam.serviceAccountTokenCreator' >/dev/null
 fi
 
 if [[ "$LABEL_ON_CRON" == "true" ]]; then

test_scripts/create_project.sh

@@ -0,0 +1,11 @@
+# Creates a project. Useful for testing. Creates the App Engine default service as well -- a prereq for Iris.
+set -u
+#set -x
+set -e
+
+gcloud projects create $PROJ --folder=$FOLDER # or else   --organization=$ORG
+gcloud billing projects link $PROJ --billing-account $BILL
+pushd helloworld_appengine ||exit
+gcloud app create --region us-west4 --project $PROJ
+gcloud app deploy -q --project $PROJ
+popd ||exit

test_scripts/helloworld_appengine/.gcloudignore

@@ -0,0 +1,19 @@
+# This file specifies files that are *not* uploaded to Google Cloud
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+# Python pycache:
+__pycache__/
+# Ignored by the build system
+/setup.cfg

test_scripts/helloworld_appengine/app.yaml

@@ -0,0 +1 @@
+runtime: python39

test_scripts/helloworld_appengine/main.py

@@ -0,0 +1,18 @@
+from flask import Flask
+
+
+app = Flask(__name__)
+
+
+@app.route('/', defaults={'path': ''})
+@app.route('/<path:path>')
+def catch_all(path):
+    return 'You want path: %s' % path
+
+@app.errorhandler(404)
+def handle_404(e):
+    # handle all other routes here
+    return f'Not Found {e}, but we HANDLED IT'
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=80)

test_scripts/helloworld_appengine/requirements.txt

@@ -0,0 +1,5 @@
+Flask
+
+
+
+