diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index a5103a836..14553b212 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -79,3 +79,5 @@ ajv-formats,import,MIT,Copyright (c) 2020 Evgeny Poberezkin jszip,import,MIT,Copyright (c) 2009-2016 Stuart Knightley and other contributors @google-cloud/run,import,Apache-2.0,Copyright (c) 2023 Google LLC and other contributors google-auth-library,import,Apache-2.0,Copyright (c) 2023 Google LLC and other contributors +@google-cloud/logging,import,Apache-2.0,Copyright (c) 2023 Google LLC and other contributors +http-proxy-agent,import,MIT,Copyright (c) 2013 Nathan Rajlich diff --git a/package.json b/package.json index 7f5c03d60..411f668af 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "@aws-sdk/client-sfn": "^3.358.0", "@aws-sdk/credential-providers": "^3.358.0", "@aws-sdk/property-provider": "^3.357.0", + "@google-cloud/logging": "^10.5.0", "@google-cloud/run": "^0.6.0", "@types/datadog-metrics": "0.6.1", "ajv": "^8.12.0", @@ -82,6 +83,7 @@ "fuzzy": "^0.1.3", "glob": "7.1.4", "google-auth-library": "^8.9.0", + "http-proxy-agent": "^7.0.0", "inquirer": "^8.2.5", "inquirer-checkbox-plus-prompt": "^1.4.2", "js-yaml": "3.13.1", diff --git a/src/commands/cloud-run/__tests__/__snapshots__/flare.test.ts.snap b/src/commands/cloud-run/__tests__/__snapshots__/flare.test.ts.snap index 24b04edae..666c85cc1 100644 --- a/src/commands/cloud-run/__tests__/__snapshots__/flare.test.ts.snap +++ b/src/commands/cloud-run/__tests__/__snapshots__/flare.test.ts.snap @@ -44,7 +44,8 @@ GCP credentials verified! } } -šŸ’¾ Saving configuration... +šŸ’¾ Saving files to mock-folder/.datadog-ci... +ā€¢ Saved function config to ./service_config.json šŸš« The flare files were not sent as it was executed in dry run mode. ā„¹ļø Your output files are located at: mock-folder/.datadog-ci @@ -64,6 +65,47 @@ GCP credentials verified! " `; +exports[`cloud-run flare getLogs handles httpRequest payload correctly 1`] = ` +Array [ + Object { + "logName": "mock-logname", + "message": "\\"GET 200. responseSize: 1.27 KB. latency: 1500 ms. requestUrl: /test-endpoint\\"", + "severity": "DEFAULT", + "timestamp": "2023-07-28 00:00:00", + }, +] +`; + +exports[`cloud-run flare getLogs handles textPayload correctly 1`] = ` +Array [ + Object { + "logName": "mock-logname", + "message": "\\"Some text payload\\"", + "severity": "DEFAULT", + "timestamp": "2023-07-28 00:00:00", + }, +] +`; + +exports[`cloud-run flare getLogs handles when a log is an HTTP request and has a textPayload 1`] = ` +Array [ + Object { + "logName": "mock-logname", + "message": "\\"Test log 1\\"", + "severity": "DEFAULT", + "timestamp": "2023-07-28 00:00:00", + }, + Object { + "logName": "mock-logname", + "message": "\\"some text payload. 504. responseSize: 0 Bytes. latency: unknown ms. requestUrl: \\"", + "severity": "", + "timestamp": "2023-07-28 00:00:01", + }, +] +`; + +exports[`cloud-run flare getLogs throws an error when \`getEntries\` fails 1`] = `[Error: getEntries failed]`; + exports[`cloud-run flare maskConfig should mask a Cloud Run config correctly 1`] = ` Object { "template": Object { @@ -153,7 +195,8 @@ GCP credentials verified! } } -šŸ’¾ Saving configuration... +šŸ’¾ Saving files to mock-folder/.datadog-ci... +ā€¢ Saved function config to ./service_config.json šŸš« The flare files were not sent based on your selection. @@ -191,7 +234,8 @@ GCP credentials verified! } } -šŸ’¾ Saving configuration... +šŸ’¾ Saving files to mock-folder/.datadog-ci... +ā€¢ Saved function config to ./service_config.json šŸš€ Sending to Datadog Support... @@ -271,7 +315,8 @@ GCP credentials verified! } } -šŸ’¾ Saving configuration... +šŸ’¾ Saving files to mock-folder/.datadog-ci... +ā€¢ Saved function config to ./service_config.json šŸš€ Sending to Datadog Support... @@ -309,7 +354,8 @@ GCP credentials verified! } } -šŸ’¾ Saving configuration... +šŸ’¾ Saving files to mock-folder/.datadog-ci... +ā€¢ Saved function config to ./service_config.json šŸš€ Sending to Datadog Support... @@ -347,7 +393,8 @@ GCP credentials verified! } } -šŸ’¾ Saving configuration... +šŸ’¾ Saving files to mock-folder/.datadog-ci... +ā€¢ Saved function config to ./service_config.json šŸš€ Sending to Datadog Support... @@ -381,7 +428,8 @@ GCP credentials verified! } } -šŸ’¾ Saving configuration... +šŸ’¾ Saving files to mock-folder/.datadog-ci... +ā€¢ Saved function config to ./service_config.json šŸš€ Sending to Datadog Support... diff --git a/src/commands/cloud-run/__tests__/flare.test.ts b/src/commands/cloud-run/__tests__/flare.test.ts index a615c94ba..e7ea10b57 100644 --- a/src/commands/cloud-run/__tests__/flare.test.ts +++ b/src/commands/cloud-run/__tests__/flare.test.ts @@ -2,6 +2,7 @@ import fs from 'fs' import process from 'process' import stream from 'stream' +import {Logging} from '@google-cloud/logging' import {GoogleAuth} from 'google-auth-library' import {API_KEY_ENV_VAR, CI_API_KEY_ENV_VAR} from '../../../constants' @@ -11,21 +12,25 @@ import { MOCK_DATADOG_API_KEY, MOCK_FLARE_FOLDER_PATH, } from '../../../helpers/__tests__/fixtures' +import * as fsModule from '../../../helpers/fs' import * as helpersPromptModule from '../../../helpers/prompt' import * as flareModule from '../flare' -import {checkAuthentication, getCloudRunServiceConfig, maskConfig} from '../flare' +import {checkAuthentication, getCloudRunServiceConfig, getLogs, maskConfig, MAX_LOGS, saveLogsFile} from '../flare' import {makeCli} from './fixtures' const MOCK_REGION = 'us-east1' +const MOCK_SERVICE = 'mock-service' +const MOCK_PROJECT = 'mock-project' +const MOCK_LOG_CLIENT = new Logging({projectId: MOCK_PROJECT}) const MOCK_REQUIRED_FLAGS = [ 'cloud-run', 'flare', '-s', - 'service', + MOCK_SERVICE, '-p', - 'project', + MOCK_PROJECT, '-r', MOCK_REGION, '-c', @@ -86,6 +91,8 @@ jest.mock('@google-cloud/run', () => { jest.spyOn(helpersPromptModule, 'requestConfirmation').mockResolvedValue(true) jest.mock('util') jest.mock('jszip') +jest.mock('@google-cloud/logging') +jest.useFakeTimers({now: new Date(Date.UTC(2023, 0))}) // File system mocks process.cwd = jest.fn().mockReturnValue(MOCK_CWD) @@ -127,7 +134,7 @@ describe('cloud-run flare', () => { const cli = makeCli() const context = createMockContext() const code = await cli.run( - ['cloud-run', 'flare', '-p', 'project', '-r', MOCK_REGION, '-c', '123', '-e', 'test@test.com'], + ['cloud-run', 'flare', '-p', MOCK_PROJECT, '-r', MOCK_REGION, '-c', '123', '-e', 'test@test.com'], context as any ) expect(code).toBe(1) @@ -139,7 +146,7 @@ describe('cloud-run flare', () => { const cli = makeCli() const context = createMockContext() const code = await cli.run( - ['cloud-run', 'flare', '-s', 'service', '-r', MOCK_REGION, '-c', '123', '-e', 'test@test.com'], + ['cloud-run', 'flare', '-s', MOCK_SERVICE, '-r', MOCK_REGION, '-c', '123', '-e', 'test@test.com'], context as any ) expect(code).toBe(1) @@ -151,7 +158,7 @@ describe('cloud-run flare', () => { const cli = makeCli() const context = createMockContext() const code = await cli.run( - ['cloud-run', 'flare', '-s', 'service', '-p', 'project', '-c', '123', '-e', 'test@test.com'], + ['cloud-run', 'flare', '-s', MOCK_SERVICE, '-p', MOCK_PROJECT, '-c', '123', '-e', 'test@test.com'], context as any ) expect(code).toBe(1) @@ -163,7 +170,7 @@ describe('cloud-run flare', () => { const cli = makeCli() const context = createMockContext() const code = await cli.run( - ['cloud-run', 'flare', '-s', 'service', '-p', 'project', '-r', MOCK_REGION, '-e', 'test@test.com'], + ['cloud-run', 'flare', '-s', MOCK_SERVICE, '-p', MOCK_PROJECT, '-r', MOCK_REGION, '-e', 'test@test.com'], context as any ) expect(code).toBe(1) @@ -175,7 +182,7 @@ describe('cloud-run flare', () => { const cli = makeCli() const context = createMockContext() const code = await cli.run( - ['cloud-run', 'flare', '-s', 'service', '-p', 'project', '-r', MOCK_REGION, '-c', '123'], + ['cloud-run', 'flare', '-s', MOCK_SERVICE, '-p', MOCK_PROJECT, '-r', MOCK_REGION, '-c', '123'], context as any ) expect(code).toBe(1) @@ -324,4 +331,152 @@ describe('cloud-run flare', () => { expect(output).toContain('šŸš« The flare files were not sent based on your selection.') }) }) + + describe('getLogs', () => { + const logName = 'mock-logname' + const mockLogs = [ + {metadata: {severity: 'DEFAULT', timestamp: '2023-07-28 00:00:00', logName, textPayload: 'Log 1'}}, + {metadata: {severity: 'INFO', timestamp: '2023-07-28 00:00:00', logName, textPayload: 'Log 2'}}, + {metadata: {severity: 'NOTICE', timestamp: '2023-07-28 01:01:01', logName, textPayload: 'Log 3'}}, + ] + const MOCK_GET_ENTRIES = MOCK_LOG_CLIENT.getEntries as jest.Mock + MOCK_GET_ENTRIES.mockResolvedValue([mockLogs, {pageToken: undefined}]) + const expectedOrder = 'timestamp asc' + + it('uses correct filter when `severityFilter` is unspecified', async () => { + await getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION) + const expectedFilter = `resource.labels.service_name="${MOCK_SERVICE}" AND resource.labels.location="${MOCK_REGION}" AND timestamp>="2022-12-31T00:00:00.000Z" AND (textPayload:* OR httpRequest:*)` + + expect(MOCK_LOG_CLIENT.getEntries).toHaveBeenCalledWith({ + filter: expectedFilter, + orderBy: expectedOrder, + pageSize: MAX_LOGS, + }) + }) + + it('uses correct filter when `severityFilter` is defined', async () => { + await getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION, ' AND severity>="WARNING"') + const expectedFilter = `resource.labels.service_name="${MOCK_SERVICE}" AND resource.labels.location="${MOCK_REGION}" AND timestamp>="2022-12-31T00:00:00.000Z" AND (textPayload:* OR httpRequest:*) AND severity>="WARNING"` + + expect(MOCK_LOG_CLIENT.getEntries).toHaveBeenCalledWith({ + filter: expectedFilter, + orderBy: expectedOrder, + pageSize: MAX_LOGS, + }) + }) + + it('converts logs to the CloudRunLog interface correctly', async () => { + const page1 = [ + {metadata: {severity: 'DEFAULT', timestamp: '2023-07-28 00:00:00', logName, textPayload: 'Test log'}}, + ] + MOCK_GET_ENTRIES.mockResolvedValueOnce([page1, {pageToken: undefined}]) + + const logs = await getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION) + + expect(logs).toEqual([ + { + severity: 'DEFAULT', + timestamp: '2023-07-28 00:00:00', + logName, + message: '"Test log"', + }, + ]) + }) + + it('throws an error when `getEntries` fails', async () => { + const error = new Error('getEntries failed') + MOCK_GET_ENTRIES.mockRejectedValueOnce(error) + + await expect(getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION)).rejects.toMatchSnapshot() + }) + + it('returns an empty array when no logs are returned', async () => { + MOCK_GET_ENTRIES.mockResolvedValueOnce([[], {pageToken: undefined}]) + const logs = await getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION) + + expect(logs).toEqual([]) + }) + + it('handles httpRequest payload correctly', async () => { + const page = [ + { + metadata: { + severity: 'DEFAULT', + timestamp: '2023-07-28 00:00:00', + logName, + httpRequest: { + requestMethod: 'GET', + status: 200, + responseSize: '1300', + latency: {seconds: '1', nanos: '500000000'}, + requestUrl: '/test-endpoint', + }, + }, + }, + ] + MOCK_GET_ENTRIES.mockResolvedValueOnce([page, {pageToken: undefined}]) + + const logs = await getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION) + expect(logs).toMatchSnapshot() + }) + + it('handles textPayload correctly', async () => { + const page = [ + { + metadata: { + severity: 'DEFAULT', + timestamp: '2023-07-28 00:00:00', + logName, + textPayload: 'Some text payload', + }, + }, + ] + MOCK_GET_ENTRIES.mockResolvedValueOnce([page, {pageToken: undefined}]) + + const logs = await getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION) + expect(logs).toMatchSnapshot() + }) + + it('handles when a log is an HTTP request and has a textPayload', async () => { + const page = [ + {metadata: {severity: 'DEFAULT', timestamp: '2023-07-28 00:00:00', logName, textPayload: 'Test log 1'}}, + { + metadata: { + httpRequest: { + status: 504, + }, + timestamp: '2023-07-28 00:00:01', + logName, + textPayload: 'some text payload.', + }, + }, + ] + MOCK_GET_ENTRIES.mockResolvedValueOnce([page, {pageToken: undefined}]) + + const logs = await getLogs(MOCK_LOG_CLIENT, MOCK_SERVICE, MOCK_REGION) + + expect(logs).toMatchSnapshot() + }) + }) + + describe('saveLogsFile', () => { + const mockLogs = [ + {severity: 'DEFAULT', timestamp: '2023-07-28 00:00:00', logName: 'mock-logname', message: 'Test log 1'}, + {severity: 'INFO', timestamp: '2023-07-28 00:00:01', logName: 'mock-logname', message: 'Test log 2'}, + {severity: 'NOTICE', timestamp: '2023-07-28 01:01:01', logName: 'mock-logname', message: 'Test log 3'}, + ] + const writeFileSpy = jest.spyOn(fsModule, 'writeFile') + const mockFilePath = 'path/to/logs.csv' + + it('should save logs to file correctly', () => { + saveLogsFile(mockLogs, mockFilePath) + const expectedContent = [ + 'severity,timestamp,logName,message', + '"DEFAULT","2023-07-28 00:00:00","mock-logname","Test log 1"', + '"INFO","2023-07-28 00:00:01","mock-logname","Test log 2"', + '"NOTICE","2023-07-28 01:01:01","mock-logname","Test log 3"', + ].join('\n') + expect(writeFileSpy).toHaveBeenCalledWith(mockFilePath, expectedContent) + }) + }) }) diff --git a/src/commands/cloud-run/flare.ts b/src/commands/cloud-run/flare.ts index 99277c271..bb90d5088 100644 --- a/src/commands/cloud-run/flare.ts +++ b/src/commands/cloud-run/flare.ts @@ -4,27 +4,47 @@ import path from 'path' import process from 'process' import util from 'util' +import {Logging} from '@google-cloud/logging' import {ServicesClient} from '@google-cloud/run' import {google} from '@google-cloud/run/build/protos/protos' import chalk from 'chalk' import {Command} from 'clipanion' import {GoogleAuth} from 'google-auth-library' -import {API_KEY_ENV_VAR, CI_API_KEY_ENV_VAR, FLARE_OUTPUT_DIRECTORY} from '../../constants' +import {API_KEY_ENV_VAR, CI_API_KEY_ENV_VAR, FLARE_OUTPUT_DIRECTORY, LOGS_DIRECTORY} from '../../constants' import {sendToDatadog} from '../../helpers/flare' import {createDirectories, deleteFolder, writeFile, zipContents} from '../../helpers/fs' import {requestConfirmation} from '../../helpers/prompt' import * as helpersRenderer from '../../helpers/renderer' -import {maskString} from '../../helpers/utils' +import {formatBytes, maskString} from '../../helpers/utils' import {SKIP_MASKING_CLOUDRUN_ENV_VARS} from './constants' +import {CloudRunLog, LogConfig} from './interfaces' import {renderAuthenticationInstructions} from './renderer' const SERVICE_CONFIG_FILE_NAME = 'service_config.json' const FLARE_ZIP_FILE_NAME = 'cloudrun-flare-output.zip' +const ALL_LOGS_FILE_NAME = 'all_logs.csv' +const WARNING_LOGS_FILE_NAME = 'warning_logs.csv' +const ERRORS_LOGS_FILE_NAME = 'error_logs.csv' +const DEBUG_LOGS_FILE_NAME = 'debug_logs.csv' + +// Must be in range 0 - 1000. If more logs are needed, pagination must be implemented +export const MAX_LOGS = 1000 +// How old the logs can be in minutes. Skip older logs +const MAX_LOG_AGE_MINUTES = 1440 +const FILTER_ORDER = 'timestamp asc' +// Types of log files to create +const LOG_CONFIGS: LogConfig[] = [ + {type: 'total', fileName: ALL_LOGS_FILE_NAME}, + {type: 'warning', severityFilter: ' AND severity>="WARNING"', fileName: WARNING_LOGS_FILE_NAME}, + {type: 'error', severityFilter: ' AND severity>="ERROR"', fileName: ERRORS_LOGS_FILE_NAME}, + {type: 'debug', severityFilter: ' AND severity="DEBUG"', fileName: DEBUG_LOGS_FILE_NAME}, +] export class CloudRunFlareCommand extends Command { private isDryRun = false + private withLogs = false private service?: string private project?: string private region?: string @@ -114,21 +134,53 @@ export class CloudRunFlareCommand extends Command { const configStr = util.inspect(config, false, 10, true) this.context.stdout.write(`\n${configStr}\n`) - // Save and zip service configuration - this.context.stdout.write(chalk.bold('\nšŸ’¾ Saving configuration...\n')) - const rootFolderPath = path.join(process.cwd(), FLARE_OUTPUT_DIRECTORY) + // Get logs + const logFileMappings = new Map() + if (this.withLogs) { + this.context.stdout.write(chalk.bold('\nšŸ“– Getting logs...\n')) + + const logClient = new Logging({projectId: this.project}) + for (const logConfig of LOG_CONFIGS) { + try { + const logs = await getLogs(logClient, this.service!, this.region!, logConfig.severityFilter) + if (logs.length === 0) { + this.context.stdout.write(`ā€¢ No ${logConfig.type} logs were found\n`) + } else { + this.context.stdout.write(`ā€¢ Found ${logs.length} ${logConfig.type} logs\n`) + logFileMappings.set(logConfig.fileName, logs) + } + } catch (err) { + const msg = err instanceof Error ? err.message : '' + this.context.stderr.write(`ā€¢ Unable to get ${logConfig.type} logs: ${msg}\n`) + } + } + } + try { - // Delete folder if it already exists + // Create folders + const rootFolderPath = path.join(process.cwd(), FLARE_OUTPUT_DIRECTORY) + const logsFolderPath = path.join(rootFolderPath, LOGS_DIRECTORY) + this.context.stdout.write(chalk.bold(`\nšŸ’¾ Saving files to ${rootFolderPath}...\n`)) if (fs.existsSync(rootFolderPath)) { deleteFolder(rootFolderPath) } + const subFolders = [] + if (logFileMappings.size > 0) { + subFolders.push(logsFolderPath) + } + createDirectories(rootFolderPath, subFolders) - // Create folder - createDirectories(rootFolderPath, []) - - // Write file + // Write config file const configFilePath = path.join(rootFolderPath, SERVICE_CONFIG_FILE_NAME) writeFile(configFilePath, JSON.stringify(config, undefined, 2)) + this.context.stdout.write(`ā€¢ Saved function config to ./${SERVICE_CONFIG_FILE_NAME}\n`) + + // Write logs + for (const [fileName, logs] of logFileMappings) { + const logFilePath = path.join(logsFolderPath, fileName) + saveLogsFile(logs, logFilePath) + this.context.stdout.write(`ā€¢ Saved logs to ./${LOGS_DIRECTORY}/${fileName}\n`) + } // Exit if dry run const outputMsg = `\nā„¹ļø Your output files are located at: ${rootFolderPath}\n\n` @@ -141,19 +193,10 @@ export class CloudRunFlareCommand extends Command { // Confirm before sending this.context.stdout.write('\n') - let confirmSendFiles - try { - confirmSendFiles = await requestConfirmation( - 'Are you sure you want to send the flare file to Datadog Support?', - false - ) - } catch (err) { - if (err instanceof Error) { - this.context.stderr.write(helpersRenderer.renderError(err.message)) - } - - return 1 - } + const confirmSendFiles = await requestConfirmation( + 'Are you sure you want to send the flare file to Datadog Support?', + false + ) if (!confirmSendFiles) { this.context.stdout.write('\nšŸš« The flare files were not sent based on your selection.') this.context.stdout.write(outputMsg) @@ -174,8 +217,10 @@ export class CloudRunFlareCommand extends Command { deleteFolder(rootFolderPath) } catch (err) { if (err instanceof Error) { - this.context.stderr.write(helpersRenderer.renderError(`Unable to save configuration: ${err.message}`)) + this.context.stderr.write(helpersRenderer.renderError(err.message)) } + + return 1 } return 0 @@ -228,16 +273,22 @@ export const getCloudRunServiceConfig = async ( */ export const maskConfig = (config: any) => { // We stringify and parse again to make a deep copy - const configCopy = JSON.parse(JSON.stringify(config)) + const configCopy: IService = JSON.parse(JSON.stringify(config)) const containers = configCopy.template?.containers if (!containers) { return configCopy } - for (const container of configCopy.template.containers) { - for (const envVar of container.env) { - if (!SKIP_MASKING_CLOUDRUN_ENV_VARS.has(envVar.name)) { - envVar.value = maskString(envVar.value) + for (const container of containers) { + const env = container.env ?? [] + for (const envVar of env) { + const name = envVar.name + const val = envVar.value + if (!name || !val) { + continue + } + if (!SKIP_MASKING_CLOUDRUN_ENV_VARS.has(name)) { + envVar.value = maskString(val) } } } @@ -245,8 +296,90 @@ export const maskConfig = (config: any) => { return configCopy } +/** + * Gets recent logs + * @param logClient Logging client + * @param serviceId + * @param location + * @param severityFilter if included, adds the string to the filter + * @returns array of logs as CloudRunLog interfaces + */ +export const getLogs = async (logClient: Logging, serviceId: string, location: string, severityFilter?: string) => { + const logs: CloudRunLog[] = [] + + // Only get recent logs + const date = new Date() + date.setMinutes(date.getMinutes() - MAX_LOG_AGE_MINUTES) + const formattedDate = date.toISOString() + + // Query options + let filter = `resource.labels.service_name="${serviceId}" AND resource.labels.location="${location}" AND timestamp>="${formattedDate}" AND (textPayload:* OR httpRequest:*)` + // We only want to get logs from the last `MAX_LOG_AGE_MINUTES` to make sure they are relevant. + // We also only want to include logs with a textPayload or logs that were an HTTP request. + // Any other logs are just audit logs which are spammy and don't have any relevant information. + filter += severityFilter ?? '' + + const options = { + filter, + orderBy: FILTER_ORDER, + pageSize: MAX_LOGS, + } + + const [entries] = await logClient.getEntries(options) + + for (const entry of entries) { + let msg = '' + if (entry.metadata.textPayload) { + msg = entry.metadata.textPayload + } + if (entry.metadata.httpRequest) { + const request = entry.metadata.httpRequest + const status = request.status ?? '' + let ms = 'unknown' + const latency = request.latency + if (latency) { + ms = (Number(latency.seconds) * 1000 + Math.round(Number(latency.nanos) / 1000000)).toString() + } + const bytes = formatBytes(Number(request.responseSize)) + const method = request.requestMethod ?? '' + const requestUrl = request.requestUrl ?? '' + msg += `${method} ${status}. responseSize: ${bytes}. latency: ${ms} ms. requestUrl: ${requestUrl}` + } + + const log: CloudRunLog = { + severity: entry.metadata.severity?.toString() ?? '', + timestamp: entry.metadata.timestamp?.toString() ?? '', + logName: entry.metadata.logName ?? '', + message: `"${msg}"`, + } + + logs.push(log) + } + + return logs +} + +/** + * Save logs in a CSV format + * @param logs array of logs stored as CloudRunLog interfaces + * @param filePath path to save the CSV file + */ +export const saveLogsFile = (logs: CloudRunLog[], filePath: string) => { + const rows = [['severity', 'timestamp', 'logName', 'message']] + logs.forEach((log) => { + const severity = `"${log.severity}"` + const timestamp = `"${log.timestamp}"` + const logName = `"${log.logName}"` + const logMessage = `"${log.message}"` + rows.push([severity, timestamp, logName, logMessage]) + }) + const data = rows.join('\n') + writeFile(filePath, data) +} + CloudRunFlareCommand.addPath('cloud-run', 'flare') CloudRunFlareCommand.addOption('isDryRun', Command.Boolean('-d,--dry')) +CloudRunFlareCommand.addOption('withLogs', Command.Boolean('--with-logs')) CloudRunFlareCommand.addOption('service', Command.String('-s,--service')) CloudRunFlareCommand.addOption('project', Command.String('-p,--project')) CloudRunFlareCommand.addOption('region', Command.String('-r,--region,-l,--location')) diff --git a/src/commands/cloud-run/interfaces.ts b/src/commands/cloud-run/interfaces.ts new file mode 100644 index 000000000..710e4517b --- /dev/null +++ b/src/commands/cloud-run/interfaces.ts @@ -0,0 +1,28 @@ +/** + * Summarize relevant information about a Cloud Run log. + * Also used to build CSV log files. + * @typedef {Object} CloudRunLog + * @property {string} severity - The level of severity of the log. It can be values such as 'DEBUG', 'INFO', 'ERROR', etc. + * @property {string} timestamp - The timestamp of when the log was generated. + * @property {string} logName - The name of the log + * @property {string} message - The actual log message detailing what event occurred. + */ +export interface CloudRunLog { + severity: string + timestamp: string + logName: string + message: string +} + +/** + * Contains all the information used to create a log file. + * @typedef {Object} LogConfig + * @property {string} type - string name of the type of log. Used when printing CLI messages. + * @property {string} fileName - The name of the log file (such as 'all_logs.csv' or 'error_logs.csv') + * @property {string} [severityFilter] - Optional filter to modify the Logging query. Example: ' AND severity="DEBUG"' + */ +export interface LogConfig { + type: string + fileName: string + severityFilter?: string +} diff --git a/src/commands/lambda/__tests__/flare.test.ts b/src/commands/lambda/__tests__/flare.test.ts index 70227a926..b3b2e0963 100644 --- a/src/commands/lambda/__tests__/flare.test.ts +++ b/src/commands/lambda/__tests__/flare.test.ts @@ -100,7 +100,7 @@ const mockJSZip = { ;(JSZip as any).mockImplementation(() => mockJSZip) // Date -jest.useFakeTimers({advanceTimers: true, now: new Date(Date.UTC(2023, 0))}) +jest.useFakeTimers({now: new Date(Date.UTC(2023, 0))}) jest.spyOn(flareModule, 'sleep').mockResolvedValue() // Misc diff --git a/src/commands/lambda/flare.ts b/src/commands/lambda/flare.ts index 622ec9127..63d912a70 100644 --- a/src/commands/lambda/flare.ts +++ b/src/commands/lambda/flare.ts @@ -14,7 +14,7 @@ import {AwsCredentialIdentity} from '@aws-sdk/types' import chalk from 'chalk' import {Command} from 'clipanion' -import {API_KEY_ENV_VAR, CI_API_KEY_ENV_VAR, FLARE_OUTPUT_DIRECTORY} from '../../constants' +import {API_KEY_ENV_VAR, CI_API_KEY_ENV_VAR, FLARE_OUTPUT_DIRECTORY, LOGS_DIRECTORY} from '../../constants' import {sendToDatadog} from '../../helpers/flare' import {createDirectories, deleteFolder, writeFile, zipContents} from '../../helpers/fs' import {requestConfirmation} from '../../helpers/prompt' @@ -34,7 +34,6 @@ import * as commonRenderer from './renderers/common-renderer' const version = require('../../../package.json').version -const LOGS_DIRECTORY = 'logs' const PROJECT_FILES_DIRECTORY = 'project_files' const ADDITIONAL_FILES_DIRECTORY = 'additional_files' const FUNCTION_CONFIG_FILE_NAME = 'function_config.json' @@ -194,6 +193,7 @@ export class LambdaFlareCommand extends Command { return 1 } + while (confirmAdditionalFiles) { this.context.stdout.write('\n') let filePath: string @@ -362,19 +362,11 @@ export class LambdaFlareCommand extends Command { // Confirm before sending this.context.stdout.write('\n') - let confirmSendFiles - try { - confirmSendFiles = await requestConfirmation( - 'Are you sure you want to send the flare file to Datadog Support?', - false - ) - } catch (err) { - if (err instanceof Error) { - this.context.stderr.write(helpersRenderer.renderError(err.message)) - } + const confirmSendFiles = await requestConfirmation( + 'Are you sure you want to send the flare file to Datadog Support?', + false + ) - return 1 - } if (!confirmSendFiles) { this.context.stdout.write('\nšŸš« The flare files were not sent based on your selection.') this.context.stdout.write(outputMsg) diff --git a/src/constants.ts b/src/constants.ts index 2399e91b5..e223601a0 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -27,4 +27,5 @@ export const SITE_ENV_VAR = 'DD_SITE' // Flare constants export const FLARE_OUTPUT_DIRECTORY = '.datadog-ci' +export const LOGS_DIRECTORY = 'logs' export const FLARE_ENDPOINT_PATH = '/api/ui/support/serverless/flare' diff --git a/yarn.lock b/yarn.lock index 936ade635..4ae485e3d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2769,6 +2769,7 @@ __metadata: "@babel/core": 7.8.0 "@babel/preset-env": 7.4.5 "@babel/preset-typescript": 7.3.3 + "@google-cloud/logging": ^10.5.0 "@google-cloud/run": ^0.6.0 "@microsoft/eslint-formatter-sarif": ^3.0.0 "@types/async-retry": 1.4.2 @@ -2814,6 +2815,7 @@ __metadata: fuzzy: ^0.1.3 glob: 7.1.4 google-auth-library: ^8.9.0 + http-proxy-agent: ^7.0.0 inquirer: ^8.2.5 inquirer-checkbox-plus-prompt: ^1.4.2 jest: 28.1.3 @@ -2946,6 +2948,70 @@ __metadata: languageName: node linkType: hard +"@google-cloud/common@npm:^4.0.0": + version: 4.0.3 + resolution: "@google-cloud/common@npm:4.0.3" + dependencies: + "@google-cloud/projectify": ^3.0.0 + "@google-cloud/promisify": ^3.0.0 + arrify: ^2.0.1 + duplexify: ^4.1.1 + ent: ^2.2.0 + extend: ^3.0.2 + google-auth-library: ^8.0.2 + retry-request: ^5.0.0 + teeny-request: ^8.0.0 + checksum: 2660da8da2295f2792a7eaa08579d3c76274b58c5d5cd652f7e242f8e593948f753925790340029db383144780b35e7ae09c3088ddbffe3dcfab950e5850de89 + languageName: node + linkType: hard + +"@google-cloud/logging@npm:^10.5.0": + version: 10.5.0 + resolution: "@google-cloud/logging@npm:10.5.0" + dependencies: + "@google-cloud/common": ^4.0.0 + "@google-cloud/paginator": ^4.0.0 + "@google-cloud/projectify": ^3.0.0 + "@google-cloud/promisify": ^3.0.0 + arrify: ^2.0.1 + dot-prop: ^6.0.0 + eventid: ^2.0.0 + extend: ^3.0.2 + gcp-metadata: ^4.0.0 + google-auth-library: ^8.0.2 + google-gax: ^3.5.8 + on-finished: ^2.3.0 + pumpify: ^2.0.1 + stream-events: ^1.0.5 + uuid: ^9.0.0 + checksum: 8847fbe7acffa599137cc470d453e9b1fada8875e9ceefb03b4523b0aeba8491e74690390f6bbd672d402b7bf0de638d34c92dfcab8fd7b9e0a89421b8887770 + languageName: node + linkType: hard + +"@google-cloud/paginator@npm:^4.0.0": + version: 4.0.1 + resolution: "@google-cloud/paginator@npm:4.0.1" + dependencies: + arrify: ^2.0.0 + extend: ^3.0.2 + checksum: 40ecfb59512ddbb76ca377cb96b61673d8d210397723dcaac41d8a553264bf0c09d3754db25dd3c476f8d85941b5017cc158b4e81c8c6a054aea020c32a1e4ba + languageName: node + linkType: hard + +"@google-cloud/projectify@npm:^3.0.0": + version: 3.0.0 + resolution: "@google-cloud/projectify@npm:3.0.0" + checksum: 4fa7ad689422b0b9c152fb00260e54e39d81678f9c51518bdb34bc57ee00604524fcdd5837fa97eb2f8ff4811afee3f345b1b0993bc4a2fa1b803bdd6554839a + languageName: node + linkType: hard + +"@google-cloud/promisify@npm:^3.0.0": + version: 3.0.1 + resolution: "@google-cloud/promisify@npm:3.0.1" + checksum: 44b4de760425d6ea328f6208c46219cfcc44383b4015c67a6b18b55b8fee5b754a11f80ed481a7d779bc471950b2b856dce51e36e8004b0d2f73a93e50d756ce + languageName: node + linkType: hard + "@google-cloud/run@npm:^0.6.0": version: 0.6.0 resolution: "@google-cloud/run@npm:0.6.0" @@ -4396,7 +4462,7 @@ __metadata: languageName: node linkType: hard -"arrify@npm:^2.0.0": +"arrify@npm:^2.0.0, arrify@npm:^2.0.1": version: 2.0.1 resolution: "arrify@npm:2.0.1" checksum: 067c4c1afd182806a82e4c1cb8acee16ab8b5284fbca1ce29408e6e91281c36bb5b612f6ddfbd40a0f7a7e0c75bf2696eb94c027f6e328d6e9c52465c98e4209 @@ -5469,7 +5535,16 @@ __metadata: languageName: node linkType: hard -"duplexify@npm:^4.0.0": +"dot-prop@npm:^6.0.0": + version: 6.0.1 + resolution: "dot-prop@npm:6.0.1" + dependencies: + is-obj: ^2.0.0 + checksum: 0f47600a4b93e1dc37261da4e6909652c008832a5d3684b5bf9a9a0d3f4c67ea949a86dceed9b72f5733ed8e8e6383cc5958df3bbd0799ee317fd181f2ece700 + languageName: node + linkType: hard + +"duplexify@npm:^4.0.0, duplexify@npm:^4.1.1": version: 4.1.2 resolution: "duplexify@npm:4.1.2" dependencies: @@ -5500,6 +5575,13 @@ __metadata: languageName: node linkType: hard +"ee-first@npm:1.1.1": + version: 1.1.1 + resolution: "ee-first@npm:1.1.1" + checksum: 1b4cac778d64ce3b582a7e26b218afe07e207a0f9bfe13cc7395a6d307849cfe361e65033c3251e00c27dd060cab43014c2d6b2647676135e18b77d2d05b3f4f + languageName: node + linkType: hard + "electron-to-chromium@npm:^1.3.723": version: 1.3.737 resolution: "electron-to-chromium@npm:1.3.737" @@ -5555,6 +5637,13 @@ __metadata: languageName: node linkType: hard +"ent@npm:^2.2.0": + version: 2.2.0 + resolution: "ent@npm:2.2.0" + checksum: f588b5707d6fef36011ea10d530645912a69530a1eb0831f8708c498ac028363a7009f45cfadd28ceb4dafd9ac17ec15213f88d09ce239cd033cfe1328dd7d7d + languageName: node + linkType: hard + "entities@npm:~2.1.0": version: 2.1.0 resolution: "entities@npm:2.1.0" @@ -6116,6 +6205,15 @@ __metadata: languageName: node linkType: hard +"eventid@npm:^2.0.0": + version: 2.0.1 + resolution: "eventid@npm:2.0.1" + dependencies: + uuid: ^8.0.0 + checksum: d7de09a0792127796d8ff413e972c0fd2bf52547fd38b7d67bc6dcb10464eb6508f60b92b60e118ecce035586326b2e159be3cec7074fcd5e0e0217c754db3be + languageName: node + linkType: hard + "execa@npm:^5.0.0": version: 5.1.1 resolution: "execa@npm:5.1.1" @@ -6494,6 +6592,19 @@ __metadata: languageName: node linkType: hard +"gaxios@npm:^4.0.0": + version: 4.3.3 + resolution: "gaxios@npm:4.3.3" + dependencies: + abort-controller: ^3.0.0 + extend: ^3.0.2 + https-proxy-agent: ^5.0.0 + is-stream: ^2.0.0 + node-fetch: ^2.6.7 + checksum: 0b72a00875404e2c3d7aca9f32535e931d7b0ebb850dc92fafc1685b99a109b04205c63e4637a2d0d9a261ac50adf83f7d33435f73e256dcca32564ef9358fee + languageName: node + linkType: hard + "gaxios@npm:^5.0.0, gaxios@npm:^5.0.1": version: 5.1.3 resolution: "gaxios@npm:5.1.3" @@ -6506,6 +6617,16 @@ __metadata: languageName: node linkType: hard +"gcp-metadata@npm:^4.0.0": + version: 4.3.1 + resolution: "gcp-metadata@npm:4.3.1" + dependencies: + gaxios: ^4.0.0 + json-bigint: ^1.0.0 + checksum: b0b1b85ea2efee1d640a1d4ead0937fdcceffd43ab4cacfdd66fd086fcfe5c3d09ad850ee14f43f2dc73244b2617b166adfa09a2a85e0652a8c56bed194f01fe + languageName: node + linkType: hard + "gcp-metadata@npm:^5.3.0": version: 5.3.0 resolution: "gcp-metadata@npm:5.3.0" @@ -7300,6 +7421,13 @@ __metadata: languageName: node linkType: hard +"is-obj@npm:^2.0.0": + version: 2.0.0 + resolution: "is-obj@npm:2.0.0" + checksum: c9916ac8f4621962a42f5e80e7ffdb1d79a3fab7456ceaeea394cd9e0858d04f985a9ace45be44433bf605673c8be8810540fe4cc7f4266fc7526ced95af5a08 + languageName: node + linkType: hard + "is-path-inside@npm:^3.0.3": version: 3.0.3 resolution: "is-path-inside@npm:3.0.3" @@ -8784,7 +8912,7 @@ jschardet@latest: languageName: node linkType: hard -"node-fetch@npm:^2.6.1, node-fetch@npm:^2.6.9": +"node-fetch@npm:^2.6.1, node-fetch@npm:^2.6.7, node-fetch@npm:^2.6.9": version: 2.6.12 resolution: "node-fetch@npm:2.6.12" dependencies: @@ -9003,6 +9131,15 @@ jschardet@latest: languageName: node linkType: hard +"on-finished@npm:^2.3.0": + version: 2.4.1 + resolution: "on-finished@npm:2.4.1" + dependencies: + ee-first: 1.1.1 + checksum: d20929a25e7f0bb62f937a425b5edeb4e4cde0540d77ba146ec9357f00b0d497cdb3b9b05b9c8e46222407d1548d08166bff69cc56dfa55ba0e4469228920ff0 + languageName: node + linkType: hard + "once@npm:^1.3.0, once@npm:^1.3.1, once@npm:^1.4.0": version: 1.4.0 resolution: "once@npm:1.4.0" @@ -9548,6 +9685,17 @@ jschardet@latest: languageName: node linkType: hard +"pumpify@npm:^2.0.1": + version: 2.0.1 + resolution: "pumpify@npm:2.0.1" + dependencies: + duplexify: ^4.1.1 + inherits: ^2.0.3 + pump: ^3.0.0 + checksum: cfc96f5307ee828ef8e6eca9fe9e1ae1de0a23ca55688bfe71ea376bc126418073dab870f02b433617f421c4545726b39e31295fce9a99b78bda5f0e527a7c11 + languageName: node + linkType: hard + "punycode@npm:^2.1.0": version: 2.1.1 resolution: "punycode@npm:2.1.1" @@ -10272,6 +10420,15 @@ jschardet@latest: languageName: node linkType: hard +"stream-events@npm:^1.0.5": + version: 1.0.5 + resolution: "stream-events@npm:1.0.5" + dependencies: + stubs: ^3.0.0 + checksum: 969ce82e34bfbef5734629cc06f9d7f3705a9ceb8fcd6a526332f9159f1f8bbfdb1a453f3ced0b728083454f7706adbbe8428bceb788a0287ca48ba2642dc3fc + languageName: node + linkType: hard + "stream-meter@npm:^1.0.4": version: 1.0.4 resolution: "stream-meter@npm:1.0.4" @@ -10447,6 +10604,13 @@ jschardet@latest: languageName: node linkType: hard +"stubs@npm:^3.0.0": + version: 3.0.0 + resolution: "stubs@npm:3.0.0" + checksum: dec7b82186e3743317616235c59bfb53284acc312cb9f4c3e97e2205c67a5c158b0ca89db5927e52351582e90a2672822eeaec9db396e23e56893d2a8676e024 + languageName: node + linkType: hard + "supports-color@npm:^5.3.0": version: 5.5.0 resolution: "supports-color@npm:5.5.0" @@ -10543,6 +10707,19 @@ jschardet@latest: languageName: node linkType: hard +"teeny-request@npm:^8.0.0": + version: 8.0.3 + resolution: "teeny-request@npm:8.0.3" + dependencies: + http-proxy-agent: ^5.0.0 + https-proxy-agent: ^5.0.0 + node-fetch: ^2.6.1 + stream-events: ^1.0.5 + uuid: ^9.0.0 + checksum: 6682a14df3708068db147c91af5f2b2e097e2e53c03dddaef40f6f974297f2da9e6112c615af9fbc84a1685c6846b8a9e485771d1a350aa25e9ff5fcf63dd821 + languageName: node + linkType: hard + "terminal-link@npm:^2.0.0": version: 2.1.1 resolution: "terminal-link@npm:2.1.1" @@ -10971,7 +11148,7 @@ jschardet@latest: languageName: node linkType: hard -"uuid@npm:^8.3.2": +"uuid@npm:^8.0.0, uuid@npm:^8.3.2": version: 8.3.2 resolution: "uuid@npm:8.3.2" bin: