From 972e6699b791b3247971c75776a00be62daf3f66 Mon Sep 17 00:00:00 2001 From: Josh Pinkney Date: Mon, 3 Feb 2025 11:53:24 -0500 Subject: [PATCH 1/3] tests(amazonq): Add inline completion e2e tests --- .../amazonq/test/e2e/inline/inline.test.ts | 175 ++++++++++++++++++ packages/core/src/test/setupUtil.ts | 2 +- 2 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 packages/amazonq/test/e2e/inline/inline.test.ts diff --git a/packages/amazonq/test/e2e/inline/inline.test.ts b/packages/amazonq/test/e2e/inline/inline.test.ts new file mode 100644 index 00000000000..649c013bf0f --- /dev/null +++ b/packages/amazonq/test/e2e/inline/inline.test.ts @@ -0,0 +1,175 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as vscode from 'vscode' +import assert from 'assert' +import { + assertTelemetry, + closeAllEditors, + getTestWindow, + registerAuthHook, + TestFolder, + toTextEditor, + using, +} from 'aws-core-vscode/test' +import { RecommendationHandler, RecommendationService } from 'aws-core-vscode/codewhisperer' +import { Commands, globals, sleep, waitUntil } from 'aws-core-vscode/shared' +import { loginToIdC } from '../amazonq/utils/setup' + +describe('Amazon Q Inline', async function () { + let tempFolder: string + const waitOptions = { + interval: 500, + timeout: 10000, + retryOnFail: false, + } + + before(async function () { + await using(registerAuthHook('amazonq-test-account'), async () => { + await loginToIdC() + }) + }) + + beforeEach(async function () { + registerAuthHook('amazonq-test-account') + const folder = await TestFolder.create() + tempFolder = folder.path + await closeAllEditors() + }) + + afterEach(async function () { + await closeAllEditors() + }) + + async function setupEditor({ name, contents }: { name?: string; contents?: string } = {}) { + const fileName = name ?? 'test.ts' + const textContents = + contents ?? + `function fib() { + + +}` + await toTextEditor(textContents, fileName, tempFolder, { + selection: new vscode.Range(new vscode.Position(1, 4), new vscode.Position(1, 4)), + }) + } + + async function waitForRecommendations() { + const ok = await waitUntil(async () => RecommendationHandler.instance.isSuggestionVisible(), waitOptions) + if (!ok) { + assert.fail('Suggestions failed to become visible') + } + } + + async function waitForTelemetry() { + const ok = await waitUntil( + async () => + globals.telemetry.logger.query({ + metricName: 'codewhisperer_userTriggerDecision', + }).length > 0, + waitOptions + ) + if (!ok) { + assert.fail('Telemetry failed to be emitted') + } + } + + for (const [name, invokeCompletion] of [ + ['automatic', async () => await vscode.commands.executeCommand('type', { text: '\n' })], + ['manual', async () => Commands.tryExecute('aws.amazonq.invokeInlineCompletion')], + ] as const) { + describe(`${name} invoke`, async function () { + let originalEditorContents: string | undefined + + describe('supported filetypes', () => { + beforeEach(async () => { + await setupEditor() + + /** + * Allow some time between when the editor is opened and when we start typing. + * If we don't do this then the time between the initial editor selection + * and invoking the "type" command is too low, causing completion to never + * activate. AFAICT there isn't anything we can use waitUntil on here + **/ + await sleep(1000) + + await invokeCompletion() + originalEditorContents = vscode.window.activeTextEditor?.document.getText() + + // wait until the ghost text appears + await waitForRecommendations() + }) + + it(`${name} invoke accept`, async function () { + /** + * keep accepting the suggestion until the text contents change + * this is required because we have no access to the inlineSuggest panel + **/ + const suggestionAccepted = await waitUntil(async () => { + // Accept the suggestion + await vscode.commands.executeCommand('editor.action.inlineSuggest.commit') + return vscode.window.activeTextEditor?.document.getText() !== originalEditorContents + }, waitOptions) + + assert.ok(suggestionAccepted, 'Editor contents should have changed') + + await waitForTelemetry() + assertTelemetry('codewhisperer_userTriggerDecision', { + codewhispererSuggestionState: 'Accept', + }) + }) + + it(`${name} invoke reject`, async function () { + // Reject the suggestion + await vscode.commands.executeCommand('aws.amazonq.rejectCodeSuggestion') + + // Contents haven't changed + assert.deepStrictEqual(vscode.window.activeTextEditor?.document.getText(), originalEditorContents) + + await waitForTelemetry() + assertTelemetry('codewhisperer_userTriggerDecision', { + codewhispererSuggestionState: 'Reject', + }) + }) + + it(`${name} invoke discard`, async function () { + // Discard the suggestion by moving it back to the original position + const position = new vscode.Position(1, 4) + const editor = vscode.window.activeTextEditor + if (!editor) { + assert.fail('Could not find text editor') + } + editor.selection = new vscode.Selection(position, position) + + // Contents are the same + assert.deepStrictEqual(vscode.window.activeTextEditor?.document.getText(), originalEditorContents) + }) + }) + + it(`${name} invoke on unsupported filetype`, async function () { + await setupEditor({ + name: 'test.zig', + contents: `fn doSomething() void { + + }`, + }) + + /** + * Add delay between editor loading and invoking completion + * @see beforeEach in supported filetypes for more information + */ + await sleep(1000) + await invokeCompletion() + + if (name === 'automatic') { + // It should never get triggered since its not a supported file type + assert.deepStrictEqual(RecommendationService.instance.isRunning, false) + } else { + await getTestWindow().waitForMessage('currently not supported by Amazon Q inline suggestions') + } + }) + }) + } +}) diff --git a/packages/core/src/test/setupUtil.ts b/packages/core/src/test/setupUtil.ts index 538af7daa06..d0a4cd0b594 100644 --- a/packages/core/src/test/setupUtil.ts +++ b/packages/core/src/test/setupUtil.ts @@ -211,7 +211,7 @@ function maskArns(text: string) { */ export function registerAuthHook(secret: string, lambdaId = process.env['AUTH_UTIL_LAMBDA_ARN']) { return getTestWindow().onDidShowMessage((message) => { - if (message.items[0].title.match(new RegExp(proceedToBrowser))) { + if (message.items.length > 0 && message.items[0].title.match(new RegExp(proceedToBrowser))) { if (!lambdaId) { const baseMessage = 'Browser login flow was shown during testing without an authorizer function' if (process.env['AWS_TOOLKIT_AUTOMATION'] === 'local') { From 9a76366bdc951d1d28063fbe0f90ea2da8b51c99 Mon Sep 17 00:00:00 2001 From: Josh Pinkney Date: Mon, 3 Feb 2025 12:29:07 -0500 Subject: [PATCH 2/3] fixup --- packages/amazonq/test/e2e/inline/inline.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/amazonq/test/e2e/inline/inline.test.ts b/packages/amazonq/test/e2e/inline/inline.test.ts index 649c013bf0f..88f7eea674d 100644 --- a/packages/amazonq/test/e2e/inline/inline.test.ts +++ b/packages/amazonq/test/e2e/inline/inline.test.ts @@ -91,9 +91,11 @@ describe('Amazon Q Inline', async function () { * Allow some time between when the editor is opened and when we start typing. * If we don't do this then the time between the initial editor selection * and invoking the "type" command is too low, causing completion to never - * activate. AFAICT there isn't anything we can use waitUntil on here + * activate. AFAICT there isn't anything we can use waitUntil on here. + * + * note: this number is entirely arbitrary **/ - await sleep(1000) + await sleep(2000) await invokeCompletion() originalEditorContents = vscode.window.activeTextEditor?.document.getText() From 8d16b7878b2628789758adca6484a4dd3f0dba3a Mon Sep 17 00:00:00 2001 From: Josh Pinkney Date: Mon, 3 Feb 2025 13:04:47 -0500 Subject: [PATCH 3/3] fixup --- packages/amazonq/test/e2e/inline/inline.test.ts | 4 +++- packages/core/src/test/codewhisperer/testUtil.ts | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/amazonq/test/e2e/inline/inline.test.ts b/packages/amazonq/test/e2e/inline/inline.test.ts index 88f7eea674d..34463449cd2 100644 --- a/packages/amazonq/test/e2e/inline/inline.test.ts +++ b/packages/amazonq/test/e2e/inline/inline.test.ts @@ -10,6 +10,7 @@ import { closeAllEditors, getTestWindow, registerAuthHook, + resetCodeWhispererGlobalVariables, TestFolder, toTextEditor, using, @@ -37,6 +38,7 @@ describe('Amazon Q Inline', async function () { const folder = await TestFolder.create() tempFolder = folder.path await closeAllEditors() + await resetCodeWhispererGlobalVariables(false) }) afterEach(async function () { @@ -95,7 +97,7 @@ describe('Amazon Q Inline', async function () { * * note: this number is entirely arbitrary **/ - await sleep(2000) + await sleep(1000) await invokeCompletion() originalEditorContents = vscode.window.activeTextEditor?.document.getText() diff --git a/packages/core/src/test/codewhisperer/testUtil.ts b/packages/core/src/test/codewhisperer/testUtil.ts index f3b82fd3850..dfa6c932878 100644 --- a/packages/core/src/test/codewhisperer/testUtil.ts +++ b/packages/core/src/test/codewhisperer/testUtil.ts @@ -28,13 +28,15 @@ import * as model from '../../codewhisperer/models/model' import { stub } from '../utilities/stubber' import { Dirent } from 'fs' // eslint-disable-line no-restricted-imports -export async function resetCodeWhispererGlobalVariables() { +export async function resetCodeWhispererGlobalVariables(clearGlobalState: boolean = true) { vsCodeState.isIntelliSenseActive = false vsCodeState.isCodeWhispererEditing = false CodeWhispererCodeCoverageTracker.instances.clear() globals.telemetry.logger.clear() session.reset() - await globals.globalState.clear() + if (clearGlobalState) { + await globals.globalState.clear() + } await CodeSuggestionsState.instance.setSuggestionsEnabled(true) await RecommendationHandler.instance.clearInlineCompletionStates() }