PROC-471: support pipeline configuration/secret retrieval

.github/workflows/verify-vault-secrets/verifyVaultSecrets.js

@@ -1,8 +1,13 @@
-const envVarsRegex = /System\.(fetch_env!|fetch_env|get_env)\(["']([^"']+)["'](?:,\s*([^)]+))?\)/g;
+const systemEnvPattern = `System\\.(fetch_env!|fetch_env|get_env)`;
+const stringLiteralPattern = `(["']([^"']+)["'])`;
+const pipe_operator = `\\|\\>`;
+const envVarPipelinePattern = `(${stringLiteralPattern}(\\s+(${pipe_operator}\\s+[.\\w\\!\\?]+\\(.*\\))|(\\s*#\\s+.+)?)*\\s+(${pipe_operator})\\s+${systemEnvPattern}\\(\\))`;
+const envVarStandardCallPattern = `(${systemEnvPattern}\\(${stringLiteralPattern}+(?:,\\s*([^)]+))?\\))`;
+const envVarsRegex = new RegExp(`${envVarPipelinePattern}|${envVarStandardCallPattern}`, "gm");
 
 function extractReferencedEnvVars(fileData, ignoredKeys) {
   const matches = fileData.matchAll(envVarsRegex);
-  const extractedEnvVars = Array.from(matches, (match) => match[2]);
+  const extractedEnvVars = Array.from(matches, (match) => match[3] ?? match[12]);
 
   return extractedEnvVars.filter((envVar) => !ignoredKeys.includes(envVar))
 }

.github/workflows/verify-vault-secrets/verifyVaultSecrets.spec.js

@@ -9,6 +9,22 @@ describe('extractReferencedEnvVars', () => {
     expect(extractedEnvVars).toEqual(['DATABASE_URL', 'API_KEY', 'API_KEY_2']);
   });
 
+  test('should extract referenced environment variables when the pipe syntax is used', () => {
+    const fileContent = `
+    "DATABASE_URL" |> System.fetch_env!()
+    "API_KEY" |> String.trim_leading("_") |> System.fetch_env()
+    "API_KEY_2" # should match pipelines like this, too, even with comments, as long as
+      |> baz() #
+      |> foo_bar() # they start with a string literal
+      # and even if they are broken by comments in between lines
+      |> System.fetch_env!() # and with comments after the env call
+      |> other_func()  # and if they are piped afterwards`;
+
+    const extractedEnvVars = extractReferencedEnvVars(fileContent, []);
+
+    expect(extractedEnvVars).toEqual(['DATABASE_URL', 'API_KEY', 'API_KEY_2']);
+  });
+
   test('should filter out ignored environment variables', () => {
     const runtimeContent = 'System.fetch_env!("DATABASE_URL") + System.fetch_env!("API_KEY")';
     const ignoredKeys = ['API_KEY'];