-
Notifications
You must be signed in to change notification settings - Fork 59
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[cloud-run][flare] Get recent logs (#1011)
* Get cloud run logs * Add pagination to get more than 1k logs * Refactor, add error handling, and tests for `getLogs()` * Add tests for `saveLogsFile()` * Fix dependency conflicts by explicitly adding the `agent-base` and `http-proxy-agent` packages. * Add new packages to LICENSE-3rdparty.csv * Fix license * Use `@google-cloud/logging-min` instead of `@google-cloud/logging` * Fix test * Refactor getting logs so and error doesn't end the program. Just skip * Refactor, fix tests * Add test * Fix `lambda/flare.ts` prompt try/catch * Refactor * Fix warnings * Skip logs that aren't HTTP requests and don't have a text payload * Fix lint * Refactor request limit checking * Refactor * Resolve * Nit * Remove unnecessary `advanceTimers` * - Use `@google-cloud/logging` instead of `@google-cloud/logging-min`. - Add JSdoc headers to interfaces.ts - Fix `logFileMappings` to have the fileName be the key instead of the value - Nits * Update LICENSE-3rdparty.csv * Remove pagination * Update comment * Update src/commands/cloud-run/interfaces.ts Co-authored-by: jordan gonzález <[email protected]> * Remove `agent-base` package * Update license --------- Co-authored-by: jordan gonzález <[email protected]>
- Loading branch information
1 parent
8f211e1
commit 113a652
Showing
10 changed files
with
601 additions
and
63 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,7 @@ import fs from 'fs' | |
| import process from 'process' | ||
| import stream from 'stream' | ||
|
|
||
| import {Logging} from '@google-cloud/logging' | ||
| import {GoogleAuth} from 'google-auth-library' | ||
|
|
||
| import {API_KEY_ENV_VAR, CI_API_KEY_ENV_VAR} from '../../../constants' | ||
|
|
@@ -11,21 +12,25 @@ import { | |
| MOCK_DATADOG_API_KEY, | ||
| MOCK_FLARE_FOLDER_PATH, | ||
| } from '../../../helpers/__tests__/fixtures' | ||
| import * as fsModule from '../../../helpers/fs' | ||
| import * as helpersPromptModule from '../../../helpers/prompt' | ||
|
|
||
| import * as flareModule from '../flare' | ||
| import {checkAuthentication, getCloudRunServiceConfig, maskConfig} from '../flare' | ||
| import {checkAuthentication, getCloudRunServiceConfig, getLogs, maskConfig, MAX_LOGS, saveLogsFile} from '../flare' | ||
|
|
||
| import {makeCli} from './fixtures' | ||
|
|
||
| const MOCK_REGION = 'us-east1' | ||
| const MOCK_SERVICE = 'mock-service' | ||
| const MOCK_PROJECT = 'mock-project' | ||
| const MOCK_LOG_CLIENT = new Logging({projectId: MOCK_PROJECT}) | ||
| const MOCK_REQUIRED_FLAGS = [ | ||
| 'cloud-run', | ||
| 'flare', | ||
| '-s', | ||
| 'service', | ||
| MOCK_SERVICE, | ||
| '-p', | ||
| 'project', | ||
| MOCK_PROJECT, | ||
| '-r', | ||
| MOCK_REGION, | ||
| '-c', | ||
|
|
@@ -86,6 +91,8 @@ jest.mock('@google-cloud/run', () => { | |
| jest.spyOn(helpersPromptModule, 'requestConfirmation').mockResolvedValue(true) | ||
| jest.mock('util') | ||
| jest.mock('jszip') | ||
| jest.mock('@google-cloud/logging') | ||
| jest.useFakeTimers({now: new Date(Date.UTC(2023, 0))}) | ||
|
|
||
| // File system mocks | ||
| process.cwd = jest.fn().mockReturnValue(MOCK_CWD) | ||
|
|
@@ -127,7 +134,7 @@ describe('cloud-run flare', () => { | |
| const cli = makeCli() | ||
| const context = createMockContext() | ||
| const code = await cli.run( | ||
| ['cloud-run', 'flare', '-p', 'project', '-r', MOCK_REGION, '-c', '123', '-e', '[email protected]'], | ||
| ['cloud-run', 'flare', '-p', MOCK_PROJECT, '-r', MOCK_REGION, '-c', '123', '-e', '[email protected]'], | ||
| context as any | ||
| ) | ||
| expect(code).toBe(1) | ||
|
|
@@ -139,7 +146,7 @@ describe('cloud-run flare', () => { | |
| const cli = makeCli() | ||
| const context = createMockContext() | ||
| const code = await cli.run( | ||
| ['cloud-run', 'flare', '-s', 'service', '-r', MOCK_REGION, '-c', '123', '-e', '[email protected]'], | ||
| ['cloud-run', 'flare', '-s', MOCK_SERVICE, '-r', MOCK_REGION, '-c', '123', '-e', '[email protected]'], | ||
| context as any | ||
| ) | ||
| expect(code).toBe(1) | ||
|
|
@@ -151,7 +158,7 @@ describe('cloud-run flare', () => { | |
| const cli = makeCli() | ||
| const context = createMockContext() | ||
| const code = await cli.run( | ||
| ['cloud-run', 'flare', '-s', 'service', '-p', 'project', '-c', '123', '-e', '[email protected]'], | ||
| ['cloud-run', 'flare', '-s', MOCK_SERVICE, '-p', MOCK_PROJECT, '-c', '123', '-e', '[email protected]'], | ||
| context as any | ||
| ) | ||
| expect(code).toBe(1) | ||
|
|
@@ -163,7 +170,7 @@ describe('cloud-run flare', () => { | |
| const cli = makeCli() | ||
| const context = createMockContext() | ||
| const code = await cli.run( | ||
| ['cloud-run', 'flare', '-s', 'service', '-p', 'project', '-r', MOCK_REGION, '-e', '[email protected]'], | ||
| ['cloud-run', 'flare', '-s', MOCK_SERVICE, '-p', MOCK_PROJECT, '-r', MOCK_REGION, '-e', '[email protected]'], | ||
| context as any | ||
| ) | ||
| expect(code).toBe(1) | ||
|
|
@@ -175,7 +182,7 @@ describe('cloud-run flare', () => { | |
| const cli = makeCli() | ||
| const context = createMockContext() | ||
| const code = await cli.run( | ||
| ['cloud-run', 'flare', '-s', 'service', '-p', 'project', '-r', MOCK_REGION, '-c', '123'], | ||
| ['cloud-run', 'flare', '-s', MOCK_SERVICE, '-p', MOCK_PROJECT, '-r', MOCK_REGION, '-c', '123'], | ||
| context as any | ||
| ) | ||
| expect(code).toBe(1) | ||
|
|
@@ -324,4 +331,152 @@ describe('cloud-run flare', () => { | |
| expect(output).toContain('🚫 The flare files were not sent based on your selection.') | ||
| }) | ||
| }) | ||
|
|
||
| describe('getLogs', () => { | ||
| const logName = 'mock-logname' | ||
| const mockLogs = [ | ||
| {metadata: {severity: 'DEFAULT', timestamp: '2023-07-28 00:00:00', logName, textPayload: 'Log 1'}}, | ||
| {metadata: {severity: 'INFO', timestamp: '2023-07-28 00:00:00', logName, textPayload: 'Log 2'}}, | ||
| {metadata: {severity: 'NOTICE', timestamp: '2023-07-28 01:01:01', logName, textPayload: 'Log 3'}}, | ||
| ] | ||
| const MOCK_GET_ENTRIES = MOCK_LOG_CLIENT.getEntries as jest.Mock | ||
| MOCK_GET_ENTRIES.mockResolvedValue([mockLogs, {pageToken: undefined}]) | ||
| const expectedOrder = 'timestamp asc' | ||
|
|
||
| it('uses correct filter when `severityFilter` is unspecified', async () => { | ||
| await getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION) | ||
| const expectedFilter = `resource.labels.service_name="${MOCK_SERVICE}" AND resource.labels.location="${MOCK_REGION}" AND timestamp>="2022-12-31T00:00:00.000Z" AND (textPayload:* OR httpRequest:*)` | ||
|
|
||
| expect(MOCK_LOG_CLIENT.getEntries).toHaveBeenCalledWith({ | ||
| filter: expectedFilter, | ||
| orderBy: expectedOrder, | ||
| pageSize: MAX_LOGS, | ||
| }) | ||
| }) | ||
|
|
||
| it('uses correct filter when `severityFilter` is defined', async () => { | ||
| await getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION, ' AND severity>="WARNING"') | ||
| const expectedFilter = `resource.labels.service_name="${MOCK_SERVICE}" AND resource.labels.location="${MOCK_REGION}" AND timestamp>="2022-12-31T00:00:00.000Z" AND (textPayload:* OR httpRequest:*) AND severity>="WARNING"` | ||
|
|
||
| expect(MOCK_LOG_CLIENT.getEntries).toHaveBeenCalledWith({ | ||
| filter: expectedFilter, | ||
| orderBy: expectedOrder, | ||
| pageSize: MAX_LOGS, | ||
| }) | ||
| }) | ||
|
|
||
| it('converts logs to the CloudRunLog interface correctly', async () => { | ||
| const page1 = [ | ||
| {metadata: {severity: 'DEFAULT', timestamp: '2023-07-28 00:00:00', logName, textPayload: 'Test log'}}, | ||
| ] | ||
| MOCK_GET_ENTRIES.mockResolvedValueOnce([page1, {pageToken: undefined}]) | ||
|
|
||
| const logs = await getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION) | ||
|
|
||
| expect(logs).toEqual([ | ||
| { | ||
| severity: 'DEFAULT', | ||
| timestamp: '2023-07-28 00:00:00', | ||
| logName, | ||
| message: '"Test log"', | ||
| }, | ||
| ]) | ||
| }) | ||
|
|
||
| it('throws an error when `getEntries` fails', async () => { | ||
| const error = new Error('getEntries failed') | ||
| MOCK_GET_ENTRIES.mockRejectedValueOnce(error) | ||
|
|
||
| await expect(getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION)).rejects.toMatchSnapshot() | ||
| }) | ||
|
|
||
| it('returns an empty array when no logs are returned', async () => { | ||
| MOCK_GET_ENTRIES.mockResolvedValueOnce([[], {pageToken: undefined}]) | ||
| const logs = await getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION) | ||
|
|
||
| expect(logs).toEqual([]) | ||
| }) | ||
|
|
||
| it('handles httpRequest payload correctly', async () => { | ||
| const page = [ | ||
| { | ||
| metadata: { | ||
| severity: 'DEFAULT', | ||
| timestamp: '2023-07-28 00:00:00', | ||
| logName, | ||
| httpRequest: { | ||
| requestMethod: 'GET', | ||
| status: 200, | ||
| responseSize: '1300', | ||
| latency: {seconds: '1', nanos: '500000000'}, | ||
| requestUrl: '/test-endpoint', | ||
| }, | ||
| }, | ||
| }, | ||
| ] | ||
| MOCK_GET_ENTRIES.mockResolvedValueOnce([page, {pageToken: undefined}]) | ||
|
|
||
| const logs = await getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION) | ||
| expect(logs).toMatchSnapshot() | ||
| }) | ||
|
|
||
| it('handles textPayload correctly', async () => { | ||
| const page = [ | ||
| { | ||
| metadata: { | ||
| severity: 'DEFAULT', | ||
| timestamp: '2023-07-28 00:00:00', | ||
| logName, | ||
| textPayload: 'Some text payload', | ||
| }, | ||
| }, | ||
| ] | ||
| MOCK_GET_ENTRIES.mockResolvedValueOnce([page, {pageToken: undefined}]) | ||
|
|
||
| const logs = await getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION) | ||
| expect(logs).toMatchSnapshot() | ||
| }) | ||
|
|
||
| it('handles when a log is an HTTP request and has a textPayload', async () => { | ||
| const page = [ | ||
| {metadata: {severity: 'DEFAULT', timestamp: '2023-07-28 00:00:00', logName, textPayload: 'Test log 1'}}, | ||
| { | ||
| metadata: { | ||
| httpRequest: { | ||
| status: 504, | ||
| }, | ||
| timestamp: '2023-07-28 00:00:01', | ||
| logName, | ||
| textPayload: 'some text payload.', | ||
| }, | ||
| }, | ||
| ] | ||
| MOCK_GET_ENTRIES.mockResolvedValueOnce([page, {pageToken: undefined}]) | ||
|
|
||
| const logs = await getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION) | ||
|
|
||
| expect(logs).toMatchSnapshot() | ||
| }) | ||
| }) | ||
|
|
||
| describe('saveLogsFile', () => { | ||
| const mockLogs = [ | ||
| {severity: 'DEFAULT', timestamp: '2023-07-28 00:00:00', logName: 'mock-logname', message: 'Test log 1'}, | ||
| {severity: 'INFO', timestamp: '2023-07-28 00:00:01', logName: 'mock-logname', message: 'Test log 2'}, | ||
| {severity: 'NOTICE', timestamp: '2023-07-28 01:01:01', logName: 'mock-logname', message: 'Test log 3'}, | ||
| ] | ||
| const writeFileSpy = jest.spyOn(fsModule, 'writeFile') | ||
| const mockFilePath = 'path/to/logs.csv' | ||
|
|
||
| it('should save logs to file correctly', () => { | ||
| saveLogsFile(mockLogs, mockFilePath) | ||
| const expectedContent = [ | ||
| 'severity,timestamp,logName,message', | ||
| '"DEFAULT","2023-07-28 00:00:00","mock-logname","Test log 1"', | ||
| '"INFO","2023-07-28 00:00:01","mock-logname","Test log 2"', | ||
| '"NOTICE","2023-07-28 01:01:01","mock-logname","Test log 3"', | ||
| ].join('\n') | ||
| expect(writeFileSpy).toHaveBeenCalledWith(mockFilePath, expectedContent) | ||
| }) | ||
| }) | ||
| }) | ||
Oops, something went wrong.