chore: added system status check to anvil-cmg tests (#4108) (#4115)

* chore: added system status check to anvil-cmg tests (#4108)

* chore: added second system status check after tests run to ensure reliability (#4108)

* chore: check that indexing checks fail appropriately (#4108)

* chore: undo test failure check (#4108)

Author: mrtopsyt

Diff for: .github/workflows/run-playwright-tests-anvil-cmg.yml

@@ -17,6 +17,10 @@ jobs:
         run: |
             cd explorer
             npm ci
+      - name: Check Backend System Status - TEST RESULTS ARE NOT VALID IF THIS FAILS
+        run: |
+            cd explorer
+            npm run check-system-status:anvil-cmg
       - name: Install Playwright Browsers
         run: |
             cd explorer
@@ -25,6 +29,10 @@ jobs:
         run: |
           cd explorer
           npm run test:anvil-cmg
+      - name: Check backend status again - TEST RESULTS ARE NOT VALID IF THIS FAILS
+        run: |
+          cd explorer
+          npm run check-system-status:anvil-cmg
       - uses: actions/upload-artifact@v3
         if: always()
         with:

Diff for: explorer/e2e/anvil/anvil-check-system-status.ts

@@ -0,0 +1,12 @@
+import { checkIsIndexing } from "../checkIsIndexing";
+const ANVIL_CMG_SYSTEM_STATUS_URL =
+  "https://service.anvil.gi.ucsc.edu/health/progress";
+
+/**
+ * Check system status for Anvil-CMG
+ */
+async function anvilCheckIsIndexing(): Promise<void> {
+  await checkIsIndexing(ANVIL_CMG_SYSTEM_STATUS_URL);
+}
+
+anvilCheckIsIndexing();

Diff for: explorer/e2e/checkIsIndexing.ts

@@ -0,0 +1,32 @@
+/**
+ * Check the system status, including whether the backend is currently indexing
+ * @param systemStatusUrl - a url to the system status api
+ */
+export async function checkIsIndexing(systemStatusUrl: string): Promise<void> {
+  const systemStatusResponse = await fetch(systemStatusUrl);
+  if (systemStatusResponse.status != 200) {
+    console.log(
+      "ERROR: The System Status API is currently unavailable, or an incorrect url was passed. Please rerun tests when this is resolved."
+    );
+    process.exit(1);
+  }
+  const systemStatusJson = await systemStatusResponse.json();
+  const isUp = systemStatusJson.up && systemStatusJson.progress.up;
+  const isIndexing =
+    systemStatusJson.progress.unindexed_bundles > 0 ||
+    systemStatusJson.progress.unindexed_documents > 0;
+  if (!isUp) {
+    console.log(
+      "There is an issue with the backend server. Please rerun tests once this issue has been resolved. If tests have already been run, please ignore the results and try again later."
+    );
+    process.exit(1);
+  } else if (isIndexing) {
+    console.log(
+      "ERROR: The database is currently indexing, which means that tests cannot run reliably. Please rerun tests later once indexing has stopped. If tests have already been run, please ignore the results and try again after indexing has stopped."
+    );
+    process.exit(1);
+  } else {
+    console.log("The System Status is currently good!");
+    process.exit(0);
+  }
+}

Diff for: explorer/package.json

@@ -33,7 +33,8 @@
     "test:e2e": "playwright test",
     "get-cellxgene-projects-hca": "esrun ./site-config/hca-dcp/dev/scripts/get-cellxgene-projects.ts ",
     "test:anvil-cmg": "playwright test -c playwright_anvil.config.ts --trace retain-on-failure",
-    "test:anvil-catalog": "playwright test -c playwright_anvil-catalog.config.ts --trace retain-on-failure"
+    "test:anvil-catalog": "playwright test -c playwright_anvil-catalog.config.ts --trace retain-on-failure",
+    "check-system-status:anvil-cmg": "esrun e2e/anvil/anvil-check-system-status.ts"
   },
   "dependencies": {
     "@databiosphere/findable-ui": "9.1.0",