diff --git a/packages/winglang/src/commands/test/harness/api.ts b/packages/winglang/src/commands/test/harness/api.ts new file mode 100644 index 00000000000..f91a3d8ad92 --- /dev/null +++ b/packages/winglang/src/commands/test/harness/api.ts @@ -0,0 +1,19 @@ +import { ITestRunnerClient } from "@winglang/sdk/lib/std"; + +/** + * API for running wing tests. + */ +export interface ITestHarness { + /** + * Deploys the test program synthesized in the given directory and return an `ITestRunnerClient` + * that can be used to run the tests. + * @param synthDir - The directory containing the synthesized test program. + */ + deploy(synthDir: string): Promise; + + /** + * Cleans up the test harness after the tests have been run. + * @param synthDir - The directory containing the synthesized test program. + */ + cleanup(synthDir: string): Promise; +} diff --git a/packages/winglang/src/commands/test/harness/awscdk.ts b/packages/winglang/src/commands/test/harness/awscdk.ts new file mode 100644 index 00000000000..a359eb3f11d --- /dev/null +++ b/packages/winglang/src/commands/test/harness/awscdk.ts @@ -0,0 +1,48 @@ +import { readFile, rm } from "fs/promises"; +import { ITestRunnerClient } from "@winglang/sdk/lib/std"; +import { Util } from "@winglang/sdk/lib/util"; +import { ITestHarness } from "./api"; +import { withSpinner } from "../../../util"; +import { execCapture } from "../util"; + +const ENV_WING_TEST_RUNNER_FUNCTION_IDENTIFIERS_AWSCDK = "WingTestRunnerFunctionArns"; + +export class AwsCdkTestHarness implements ITestHarness { + public async deploy(synthDir: string): Promise { + try { + await execCapture("cdk version --ci true", { cwd: synthDir }); + } catch (err) { + throw new Error( + "AWS-CDK is not installed. Please install AWS-CDK to run tests in the cloud (npm i -g aws-cdk)." + ); + } + + await withSpinner("cdk deploy", () => + execCapture("cdk deploy --require-approval never --ci true -O ./output.json --app . ", { + cwd: synthDir, + }) + ); + + const stackName = process.env.CDK_STACK_NAME! + Util.sha256(synthDir).slice(-8); + const testArns = await this.getFunctionArnsOutput(synthDir, stackName); + + const { TestRunnerClient } = await import("@winglang/sdk/lib/shared-aws/test-runner.inflight"); + const runner = new TestRunnerClient({ $tests: testArns }); + return runner; + } + + public async cleanup(synthDir: string): Promise { + await withSpinner("aws-cdk destroy", async () => { + await rm(synthDir.concat("/output.json")); + await execCapture("cdk destroy -f --ci true --app ./", { cwd: synthDir }); + }); + + await rm(synthDir, { recursive: true, force: true }); + } + + private async getFunctionArnsOutput(synthDir: string, stackName: string) { + const file = await readFile(synthDir.concat("/output.json")); + const parsed = JSON.parse(Buffer.from(file).toString()); + return parsed[stackName][ENV_WING_TEST_RUNNER_FUNCTION_IDENTIFIERS_AWSCDK]; + } +} diff --git a/packages/winglang/src/commands/test/harness/terraform.ts b/packages/winglang/src/commands/test/harness/terraform.ts new file mode 100644 index 00000000000..b0863bfc7c7 --- /dev/null +++ b/packages/winglang/src/commands/test/harness/terraform.ts @@ -0,0 +1,72 @@ +import { rm } from "fs/promises"; +import { BuiltinPlatform, determineTargetFromPlatforms } from "@winglang/compiler"; +import { ITestRunnerClient } from "@winglang/sdk/lib/std"; +import { ITestHarness } from "./api"; +import { withSpinner } from "../../../util"; +import { TestOptions } from "../test"; +import { execCapture } from "../util"; + +const ENV_WING_TEST_RUNNER_FUNCTION_IDENTIFIERS = "WING_TEST_RUNNER_FUNCTION_IDENTIFIERS"; +const PARALLELISM = { [BuiltinPlatform.TF_AZURE]: 5 }; +const targetFolder: Record = { + [BuiltinPlatform.TF_AWS]: "shared-aws", + [BuiltinPlatform.TF_AZURE]: "shared-azure", + [BuiltinPlatform.TF_GCP]: "shared-gcp", +}; + +export class TerraformTestHarness implements ITestHarness { + private readonly options: TestOptions; + private readonly parallelism: string; + + constructor(options: TestOptions) { + this.options = options; + const p = PARALLELISM[options.platform[0]]; + this.parallelism = p ? `-parallelism=${p}` : ""; + } + + public async deploy(synthDir: string): Promise { + // Check if Terraform is installed + const tfVersion = await execCapture("terraform version", { cwd: synthDir }); + const installed = tfVersion.startsWith("Terraform v"); + if (!installed) { + throw new Error( + "Terraform is not installed. Please install Terraform to run tests in the cloud." + ); + } + + // Initialize Terraform + await withSpinner("terraform init", () => execCapture("terraform init", { cwd: synthDir })); + + // Apply Terraform + await withSpinner("terraform apply", () => + execCapture(`terraform apply -auto-approve ${this.parallelism}`, { cwd: synthDir }) + ); + + // Get the test runner function ARNs + const output = await execCapture("terraform output -json", { cwd: synthDir }); + const parsed = JSON.parse(output); + const testArns = parsed[ENV_WING_TEST_RUNNER_FUNCTION_IDENTIFIERS]?.value; + if (!testArns) { + throw new Error(`terraform output ${ENV_WING_TEST_RUNNER_FUNCTION_IDENTIFIERS} not found`); + } + + // Create the test runner client + const target = determineTargetFromPlatforms(this.options.platform); + const testRunnerPath = `@winglang/sdk/lib/${targetFolder[target]}/test-runner.inflight`; + const { TestRunnerClient } = await import(testRunnerPath); + const runner = new TestRunnerClient({ $tests: testArns }); + return runner; + } + + public async cleanup(synthDir: string): Promise { + try { + await withSpinner("terraform destroy", () => + execCapture(`terraform destroy -auto-approve ${this.parallelism}`, { cwd: synthDir }) + ); + + await rm(synthDir, { recursive: true, force: true }); + } catch (e) { + console.error(e); + } + } +} diff --git a/packages/winglang/src/commands/test/test.ts b/packages/winglang/src/commands/test/test.ts index f01e5d2ce45..1540d63a64c 100644 --- a/packages/winglang/src/commands/test/test.ts +++ b/packages/winglang/src/commands/test/test.ts @@ -1,17 +1,16 @@ -import * as cp from "child_process"; -import { existsSync, readFile, readFileSync, realpathSync, rm, rmSync, statSync } from "fs"; +import { existsSync, readFileSync, realpathSync, rmSync, statSync } from "fs"; import { basename, join, relative, resolve } from "path"; -import { promisify } from "util"; import { PromisePool } from "@supercharge/promise-pool"; import { BuiltinPlatform, determineTargetFromPlatforms } from "@winglang/compiler"; import { std, simulator } from "@winglang/sdk"; import { LogLevel } from "@winglang/sdk/lib/std"; -import { Util } from "@winglang/sdk/lib/util"; import { prettyPrintError } from "@winglang/sdk/lib/util/enhanced-error"; import chalk from "chalk"; -import debug from "debug"; import { glob } from "glob"; import { nanoid } from "nanoid"; +import { ITestHarness } from "./harness/api"; +import { AwsCdkTestHarness } from "./harness/awscdk"; +import { TerraformTestHarness } from "./harness/terraform"; import { printResults, validateOutputFilePath, writeResultsToFile } from "./results"; import { SnapshotMode, SnapshotResult, captureSnapshot, determineSnapshotMode } from "./snapshots"; import { SNAPSHOT_ERROR_PREFIX } from "./snapshots-help"; @@ -21,13 +20,6 @@ import { withSpinner } from "../../util"; import { compile, CompileOptions } from "../compile"; import { SpinnerStream } from "../spinner-stream"; -const log = debug("wing:test"); - -const ENV_WING_TEST_RUNNER_FUNCTION_IDENTIFIERS = "WING_TEST_RUNNER_FUNCTION_IDENTIFIERS"; -const ENV_WING_TEST_RUNNER_FUNCTION_IDENTIFIERS_AWSCDK = "WingTestRunnerFunctionArns"; - -const PARALLELISM = { [BuiltinPlatform.TF_AZURE]: 5 }; - /** * Options for the `test` command. */ @@ -232,17 +224,29 @@ async function executeTest( target: string | undefined, options: TestOptions ): Promise { + if (!target) { + throw new Error("Unable to execute test without a target"); + } + + // special case for simulator + if (target === BuiltinPlatform.SIM) { + return testSimulator(synthDir, options); + } + + const harness = createTestHarness(target, options); + return executeTestInHarness(harness, synthDir, options); +} + +function createTestHarness(target: string, options: TestOptions) { switch (target) { - case BuiltinPlatform.SIM: - return testSimulator(synthDir, options); case BuiltinPlatform.TF_AZURE: case BuiltinPlatform.TF_AWS: case BuiltinPlatform.TF_GCP: - return testTf(synthDir, options); + return new TerraformTestHarness(options); case BuiltinPlatform.AWSCDK: - return testAwsCdk(synthDir, options); + return new AwsCdkTestHarness(); default: - throw new Error(`unsupported target ${target}`); + throw new Error("Unable to create a harness for platform"); } } @@ -531,32 +535,13 @@ async function testSimulator(synthDir: string, options: TestOptions) { return results.map((r) => ({ ...r, args })); } -async function testTf(synthDir: string, options: TestOptions): Promise { - const { clean, testFilter, platform = [BuiltinPlatform.SIM] } = options; - let tfParallelism = PARALLELISM[platform[0]]; - +async function executeTestInHarness(harness: ITestHarness, synthDir: string, options: TestOptions) { try { - const installed = await isTerraformInstalled(synthDir); - if (!installed) { - throw new Error( - "Terraform is not installed. Please install Terraform to run tests in the cloud." - ); - } - - await withSpinner("terraform init", async () => terraformInit(synthDir)); - - await withSpinner("terraform apply", () => terraformApply(synthDir, tfParallelism)); + const runner = await harness.deploy(synthDir); const [testRunner, tests] = await withSpinner("Setting up test runner...", async () => { - const target = determineTargetFromPlatforms(platform); - const testRunnerPath = `@winglang/sdk/lib/${targetFolder[target]}/test-runner.inflight`; - - const testArns = await terraformOutput(synthDir, ENV_WING_TEST_RUNNER_FUNCTION_IDENTIFIERS); - const { TestRunnerClient } = await import(testRunnerPath); - const runner = new TestRunnerClient({ $tests: testArns }); - const allTests = await runner.listTests(); - const filteredTests = filterTests(allTests, testFilter); + const filteredTests = filterTests(allTests, options.testFilter); return [runner, filteredTests]; }); @@ -583,149 +568,14 @@ async function testTf(synthDir: string, options: TestOptions): Promise { - const { clean, testFilter } = options; - try { - await isAwsCdkInstalled(synthDir); - - await withSpinner("cdk deploy", () => awsCdkDeploy(synthDir)); - - const [testRunner, tests] = await withSpinner("Setting up test runner...", async () => { - const stackName = process.env.CDK_STACK_NAME! + Util.sha256(synthDir).slice(-8); - - const testArns = await awsCdkOutput( - synthDir, - ENV_WING_TEST_RUNNER_FUNCTION_IDENTIFIERS_AWSCDK, - stackName - ); - - const { TestRunnerClient } = await import( - "@winglang/sdk/lib/shared-aws/test-runner.inflight" - ); - const runner = new TestRunnerClient({ $tests: testArns }); - - const allTests = await runner.listTests(); - const filteredTests = filterTests(allTests, testFilter); - return [runner, filteredTests]; - }); - - const results = await withSpinner("Running tests...", async () => { - return runTests(testRunner, tests); - }); - - const testReport = await renderTestReport(synthDir, results); - if (testReport.length > 0) { - console.log(testReport); - } - - if (testResultsContainsFailure(results)) { - console.log("One or more tests failed. Cleaning up resources..."); - } - - return results; - } catch (err) { - console.warn((err as Error).message); - return [{ pass: false, path: "", error: (err as Error).message, traces: [] }]; - } finally { - if (clean) { - await cleanupCdk(synthDir); - } else { - noCleanUp(synthDir); - } - } -} - -async function cleanupCdk(synthDir: string) { - await withSpinner("aws-cdk destroy", () => awsCdkDestroy(synthDir)); - rmSync(synthDir, { recursive: true, force: true }); -} - -async function isAwsCdkInstalled(synthDir: string) { - try { - await execCapture("cdk version --ci true", { cwd: synthDir }); - } catch (err) { - throw new Error( - "AWS-CDK is not installed. Please install AWS-CDK to run tests in the cloud (npm i -g aws-cdk)." - ); - } -} - -export async function awsCdkDeploy(synthDir: string) { - await execCapture("cdk deploy --require-approval never --ci true -O ./output.json --app . ", { - cwd: synthDir, - }); -} - -export async function awsCdkDestroy(synthDir: string) { - const removeFile = promisify(rm); - await removeFile(synthDir.concat("/output.json")); - await execCapture("cdk destroy -f --ci true --app ./", { cwd: synthDir }); -} - -async function awsCdkOutput(synthDir: string, name: string, stackName: string) { - const readFileCmd = promisify(readFile); - const file = await readFileCmd(synthDir.concat("/output.json")); - const parsed = JSON.parse(Buffer.from(file).toString()); - return parsed[stackName][name]; -} - -const targetFolder: Record = { - [BuiltinPlatform.TF_AWS]: "shared-aws", - [BuiltinPlatform.TF_AZURE]: "shared-azure", - [BuiltinPlatform.TF_GCP]: "shared-gcp", -}; - -async function cleanupTf(synthDir: string, parallelism?: number) { - try { - await withSpinner("terraform destroy", () => terraformDestroy(synthDir, parallelism)); - rmSync(synthDir, { recursive: true, force: true }); - } catch (e) { - console.error(e); - } -} - -async function isTerraformInstalled(synthDir: string) { - const output = await execCapture("terraform version", { cwd: synthDir }); - return output.startsWith("Terraform v"); -} - -export async function terraformInit(synthDir: string) { - return execCapture("terraform init", { cwd: synthDir }); -} - -async function terraformApply(synthDir: string, parallelism?: number) { - return execCapture( - `terraform apply -auto-approve ${parallelism ? `-parallelism=${parallelism}` : ""}`, - { cwd: synthDir } - ); -} - -async function terraformDestroy(synthDir: string, parallelism?: number) { - return execCapture( - `terraform destroy -auto-approve ${parallelism ? `-parallelism=${parallelism}` : ""}`, - { - cwd: synthDir, - } - ); -} - -async function terraformOutput(synthDir: string, name: string) { - const output = await execCapture("terraform output -json", { cwd: synthDir }); - const parsed = JSON.parse(output); - if (!parsed[name]) { - throw new Error(`terraform output ${name} not found`); - } - return parsed[name].value; -} - function sortTests(a: std.TestResult, b: std.TestResult) { if (a.pass && !b.pass) { return -1; @@ -748,22 +598,3 @@ function extractTestNameFromPath(path: string): string | undefined { } return undefined; } - -const MAX_BUFFER = 10 * 1024 * 1024; - -/** - * Executes command and returns STDOUT. If the command fails (non-zero), throws an error. - */ -async function execCapture(command: string, options: { cwd: string }) { - log(command); - const exec = promisify(cp.exec); - const { stdout, stderr } = await exec(command, { - cwd: options.cwd, - maxBuffer: MAX_BUFFER, - }); - if (stderr) { - throw new Error(stderr); - } - log(stdout); - return stdout; -} diff --git a/packages/winglang/src/commands/test/util.ts b/packages/winglang/src/commands/test/util.ts index 9c838ba2b16..172b42dff14 100644 --- a/packages/winglang/src/commands/test/util.ts +++ b/packages/winglang/src/commands/test/util.ts @@ -1,7 +1,31 @@ +import * as cp from "child_process"; import { sep } from "path"; +import { promisify } from "util"; +import debug from "debug"; + +const log = debug("wing:test"); /** * @param path path to the test/s file * @returns the file name and parent dir in the following format: "folder/file.ext" */ export const renderTestName = (path: string) => path.split(sep).slice(-2).join("/"); + +const MAX_BUFFER = 10 * 1024 * 1024; + +/** + * Executes command and returns STDOUT. If the command fails (non-zero), throws an error. + */ +export async function execCapture(command: string, options: { cwd: string }) { + log(command); + const exec = promisify(cp.exec); + const { stdout, stderr } = await exec(command, { + cwd: options.cwd, + maxBuffer: MAX_BUFFER, + }); + if (stderr) { + throw new Error(stderr); + } + log(stdout); + return stdout; +}