Skip to content

Commit e28d181

Browse files
authored
test: fail on unexpected stdout/stderr in Vitest (#911)
1 parent 657cda6 commit e28d181

File tree

8 files changed

+262
-12
lines changed

8 files changed

+262
-12
lines changed

.changeset/quiet-console-tests.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
'@openai/agents-core': patch
3+
'@openai/agents-extensions': patch
4+
'@openai/agents-realtime': patch
5+
---
6+
7+
test: fail on unexpected stdout/stderr in Vitest

helpers/tests/console-guard.ts

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
import { afterEach, beforeEach, expect } from 'vitest';
2+
import { format } from 'node:util';
3+
4+
type ConsoleMethod = 'log' | 'info' | 'debug' | 'warn' | 'error';
5+
type StreamKind = 'stdout' | 'stderr';
6+
7+
type OutputEvent = {
8+
kind: StreamKind;
9+
source: 'console' | 'stream';
10+
method?: ConsoleMethod;
11+
message: string;
12+
};
13+
14+
const mode = (process.env.TEST_STDIO_MODE ?? 'error').toLowerCase();
15+
const guardEnabled = mode !== 'off';
16+
const MAX_EVENTS = 20;
17+
18+
const consoleToStream: Record<ConsoleMethod, StreamKind> = {
19+
log: 'stdout',
20+
info: 'stdout',
21+
debug: 'stdout',
22+
warn: 'stderr',
23+
error: 'stderr',
24+
};
25+
26+
const consoleMethods = Object.keys(consoleToStream) as ConsoleMethod[];
27+
const baselineConsole = new Map<ConsoleMethod, (...args: unknown[]) => void>();
28+
for (const method of consoleMethods) {
29+
baselineConsole.set(method, console[method].bind(console));
30+
}
31+
32+
const originalStdoutWrite = process.stdout.write.bind(process.stdout);
33+
const originalStderrWrite = process.stderr.write.bind(process.stderr);
34+
35+
let consoleWriteDepth = 0;
36+
let allowAll = false;
37+
let droppedEvents = 0;
38+
const allowedKinds = new Set<ConsoleMethod | StreamKind>();
39+
let outputEvents: OutputEvent[] = [];
40+
41+
function normalizeChunk(chunk: unknown, encoding?: BufferEncoding): string {
42+
if (typeof chunk === 'string') {
43+
return chunk;
44+
}
45+
if (chunk instanceof Uint8Array) {
46+
return Buffer.from(chunk).toString(encoding);
47+
}
48+
return String(chunk);
49+
}
50+
51+
function truncate(message: string, maxLength = 200): string {
52+
const trimmed = message.trimEnd();
53+
if (trimmed.length <= maxLength) {
54+
return trimmed;
55+
}
56+
return `${trimmed.slice(0, maxLength)}...`;
57+
}
58+
59+
function isAllowed(kind: StreamKind, method?: ConsoleMethod): boolean {
60+
if (allowAll) {
61+
return true;
62+
}
63+
if (allowedKinds.has(kind)) {
64+
return true;
65+
}
66+
if (method && allowedKinds.has(method)) {
67+
return true;
68+
}
69+
return false;
70+
}
71+
72+
function record(event: OutputEvent): void {
73+
if (outputEvents.length >= MAX_EVENTS) {
74+
droppedEvents += 1;
75+
return;
76+
}
77+
outputEvents.push({ ...event, message: event.message.trimEnd() });
78+
}
79+
80+
export function allowConsole(kinds?: Array<ConsoleMethod | StreamKind>): void {
81+
if (!guardEnabled) {
82+
return;
83+
}
84+
if (!kinds || kinds.length === 0) {
85+
allowAll = true;
86+
return;
87+
}
88+
for (const kind of kinds) {
89+
allowedKinds.add(kind);
90+
}
91+
}
92+
93+
function patchConsole(): void {
94+
for (const method of consoleMethods) {
95+
const original = baselineConsole.get(method);
96+
if (!original) {
97+
continue;
98+
}
99+
100+
console[method] = (...args: unknown[]) => {
101+
const kind = consoleToStream[method];
102+
const message = format(...args);
103+
104+
if (!isAllowed(kind, method)) {
105+
record({ kind, source: 'console', method, message });
106+
return;
107+
}
108+
109+
consoleWriteDepth += 1;
110+
try {
111+
original(...args);
112+
} finally {
113+
consoleWriteDepth -= 1;
114+
}
115+
};
116+
}
117+
}
118+
119+
function patchStream(kind: StreamKind): void {
120+
const stream = kind === 'stdout' ? process.stdout : process.stderr;
121+
const originalWrite =
122+
kind === 'stdout' ? originalStdoutWrite : originalStderrWrite;
123+
124+
stream.write = ((chunk: unknown, encoding?: unknown, cb?: unknown) => {
125+
const callback =
126+
typeof encoding === 'function'
127+
? (encoding as (err?: Error) => void)
128+
: (cb as ((err?: Error) => void) | undefined);
129+
const resolvedEncoding =
130+
typeof encoding === 'string' ? (encoding as BufferEncoding) : undefined;
131+
132+
// Avoid double counting console.* calls that delegate to stream writes.
133+
if (consoleWriteDepth === 0) {
134+
const message = normalizeChunk(chunk, resolvedEncoding);
135+
if (!isAllowed(kind)) {
136+
record({ kind, source: 'stream', message });
137+
}
138+
}
139+
140+
if (isAllowed(kind)) {
141+
return originalWrite(
142+
chunk as any,
143+
resolvedEncoding as any,
144+
callback as any,
145+
);
146+
}
147+
148+
callback?.();
149+
return true;
150+
}) as typeof stream.write;
151+
}
152+
153+
function patchAll(): void {
154+
if (!guardEnabled) {
155+
return;
156+
}
157+
patchConsole();
158+
patchStream('stdout');
159+
patchStream('stderr');
160+
}
161+
162+
function formatEvent(event: OutputEvent): string {
163+
const label =
164+
event.source === 'console'
165+
? `console.${event.method}`
166+
: `${event.kind}.write`;
167+
return `${label}: ${truncate(event.message)}`;
168+
}
169+
170+
function buildFailureMessage(): string {
171+
const testName = expect.getState().currentTestName ?? '<unknown test>';
172+
const lines = outputEvents.map(formatEvent);
173+
if (droppedEvents > 0) {
174+
lines.push(`...and ${droppedEvents} more event(s).`);
175+
}
176+
return [
177+
`Unexpected stdout/stderr during test: ${testName}`,
178+
...lines.map((line) => ` - ${line}`),
179+
'Use allowConsole([...]) in a test when output is intentional.',
180+
'Set TEST_STDIO_MODE=off to disable this guard locally.',
181+
].join('\n');
182+
}
183+
184+
patchAll();
185+
186+
beforeEach(() => {
187+
if (!guardEnabled) {
188+
return;
189+
}
190+
// Reapply patches in case a previous test restored mocked console methods.
191+
patchAll();
192+
allowAll = false;
193+
droppedEvents = 0;
194+
allowedKinds.clear();
195+
outputEvents = [];
196+
});
197+
198+
afterEach(() => {
199+
if (!guardEnabled || outputEvents.length === 0) {
200+
return;
201+
}
202+
const message = buildFailureMessage();
203+
if (mode === 'warn') {
204+
originalStderrWrite(`${message}\n`);
205+
return;
206+
}
207+
throw new Error(message);
208+
});

packages/agents-core/src/logger.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ export function getLogger(namespace: string = 'openai-agents'): Logger {
6060
return {
6161
namespace,
6262
debug: debug(namespace),
63-
error: console.error,
64-
warn: console.warn,
63+
error: (...args: any[]) => console.error(...args),
64+
warn: (...args: any[]) => console.warn(...args),
6565
dontLogModelData,
6666
dontLogToolData,
6767
};

packages/agents-core/test/tracing.test.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import {
2727
} from '../src/tracing/processor';
2828

2929
import coreLogger from '../src/logger';
30+
import { allowConsole } from '../../../helpers/tests/console-guard';
3031

3132
import {
3233
withTrace,
@@ -369,6 +370,7 @@ describe('Runner tracing configuration', () => {
369370

370371
describe('ConsoleSpanExporter', () => {
371372
it('skips export when tracing is disabled', async () => {
373+
allowConsole(['log']);
372374
const debugSpy = vi.spyOn(coreLogger, 'debug').mockImplementation(() => {});
373375
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
374376
setTracingDisabled(true);
@@ -387,6 +389,7 @@ describe('ConsoleSpanExporter', () => {
387389
});
388390

389391
it('logs traces and spans when tracing is enabled', async () => {
392+
allowConsole(['log']);
390393
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
391394
const originalEnv = process.env.NODE_ENV;
392395
const originalDisableTracing = process.env.OPENAI_AGENTS_DISABLE_TRACING;

packages/agents-extensions/test/TwilioRealtimeTransport.test.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { describe, test, expect, vi, beforeEach } from 'vitest';
22
import { EventEmitter } from 'events';
33
import { TwilioRealtimeTransportLayer } from '../src/TwilioRealtimeTransport';
4+
import { allowConsole } from '../../../helpers/tests/console-guard';
45

56
import type { MessageEvent as NodeMessageEvent } from 'ws';
67
import type { MessageEvent } from 'undici-types';
@@ -64,6 +65,7 @@ describe('TwilioRealtimeTransportLayer', () => {
6465
});
6566

6667
test('connect handles messages and events', async () => {
68+
allowConsole(['error']);
6769
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
6870
const twilio = new FakeTwilioWebSocket();
6971
const transport = new TwilioRealtimeTransportLayer({
@@ -175,6 +177,7 @@ describe('TwilioRealtimeTransportLayer', () => {
175177
});
176178

177179
test('resets counters on new Twilio start and handles invalid marks', async () => {
180+
allowConsole(['warn']);
178181
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
179182
const twilio = new FakeTwilioWebSocket();
180183
const transport = new TwilioRealtimeTransportLayer({

packages/agents-extensions/test/ai-sdk/index.test.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import { protocol, withTrace, UserError } from '@openai/agents';
1111
import { ReadableStream } from 'node:stream/web';
1212
import type { JSONSchema7, LanguageModelV2 } from '@ai-sdk/provider';
1313
import type { SerializedOutputType } from '@openai/agents';
14+
import { allowConsole } from '../../../../helpers/tests/console-guard';
1415

1516
function stubModel(
1617
partial: Partial<Pick<LanguageModelV2, 'doGenerate' | 'doStream'>>,
@@ -939,6 +940,7 @@ describe('AiSdkModel.getResponse', () => {
939940
});
940941

941942
test('handles function call output', async () => {
943+
allowConsole(['warn']);
942944
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
943945
const model = new AiSdkModel(
944946
stubModel({
@@ -1058,6 +1060,7 @@ describe('AiSdkModel.getResponse', () => {
10581060
});
10591061

10601062
test('falls back to result.providerMetadata when toolCall.providerMetadata is undefined', async () => {
1063+
allowConsole(['warn']);
10611064
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
10621065
const resultProviderMetadata = { fallback: true };
10631066

packages/agents-realtime/test/realtimeAgentHandoffs.test.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ describe('RealtimeAgent handoffs', () => {
6969
parameters,
7070
execute: async ({ message }, runContext) => {
7171
// if you want to access history data, the type parameter must be RealtimeContextData<SessionContext>
72-
console.log(runContext?.context?.history);
72+
void runContext?.context?.history;
7373
return `${runContext?.context?.userId}: ${message}`;
7474
},
7575
});

vitest.config.ts

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,46 @@
1+
import { readFileSync, readdirSync } from 'node:fs';
2+
import { resolve, dirname } from 'node:path';
3+
import { fileURLToPath } from 'node:url';
14
import { defineConfig } from 'vitest/config';
25

6+
const rootDir = dirname(fileURLToPath(import.meta.url));
7+
const packagesDir = resolve(rootDir, 'packages');
8+
9+
const baseTestConfig = {
10+
setupFiles: [resolve(rootDir, 'helpers/tests/console-guard.ts')],
11+
globalSetup: resolve(rootDir, 'helpers/tests/setup.ts'),
12+
};
13+
14+
const packageEntries = readdirSync(packagesDir, { withFileTypes: true }).filter(
15+
(entry) => entry.isDirectory(),
16+
);
17+
18+
const packageProjects = packageEntries.map((entry) => {
19+
const root = resolve(packagesDir, entry.name);
20+
const packageJsonPath = resolve(root, 'package.json');
21+
const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf8')) as {
22+
name?: string;
23+
};
24+
const name = packageJson.name ?? entry.name;
25+
26+
return {
27+
root,
28+
test: {
29+
...baseTestConfig,
30+
name,
31+
},
32+
};
33+
});
34+
335
export default defineConfig({
436
test: {
5-
projects: ['packages/*'],
6-
globalSetup: './helpers/tests/setup.ts',
7-
// Enable code coverage reporting with Vitest's built‑in integration. We
8-
// only enable it for the monorepo packages (workspaces) so that the
9-
// initial report focuses on our public libraries and avoids unnecessary
10-
// noise from docs and examples.
37+
projects: packageProjects,
38+
// Coverage options are global in Vitest workspaces.
39+
// Keep the filter at the root to avoid scanning docs/examples/dist output.
1140
coverage: {
1241
provider: 'v8',
1342
reporter: ['text', 'html', 'json', 'json-summary', 'lcov'],
1443
all: true,
15-
// Only include source files from the published packages. This keeps the
16-
// metrics meaningful and prevents Vitest from trying to instrument node
17-
// dependencies or the compiled dist folder.
1844
include: ['packages/**/src/**/*.ts'],
1945
exclude: ['**/*.d.ts', 'packages/**/test/**', 'packages/**/dist/**'],
2046
},

0 commit comments

Comments
 (0)