From c7ae1f51e24b27c0152df730adeec6f3bc66a2a2 Mon Sep 17 00:00:00 2001 From: ai16z-demirix Date: Thu, 12 Dec 2024 01:00:06 +0100 Subject: [PATCH] test: adding tests. changed files actions.test.ts, messages.test.ts, models.test.ts --- packages/core/src/tests/actions.test.ts | 168 +++++++++++-------- packages/core/src/tests/messages.test.ts | 198 ++++++++++++++++++++++- packages/core/src/tests/models.test.ts | 140 +++++++++++++--- 3 files changed, 408 insertions(+), 98 deletions(-) diff --git a/packages/core/src/tests/actions.test.ts b/packages/core/src/tests/actions.test.ts index f3ac2e32dee..ab0fcdfb915 100644 --- a/packages/core/src/tests/actions.test.ts +++ b/packages/core/src/tests/actions.test.ts @@ -4,13 +4,7 @@ import { formatActionNames, formatActions, } from "../actions"; -import { - Action, - HandlerCallback, - IAgentRuntime, - Memory, - State, -} from "../types"; +import { Action } from "../types"; describe("Actions", () => { const mockActions: Action[] = [ @@ -25,24 +19,14 @@ describe("Actions", () => { content: { text: "Hi {{user1}}!", action: "wave" }, }, ], + [ + { user: "user1", content: { text: "Hey {{user2}}, how are you?" } }, + { user: "user2", content: { text: "I'm good {{user1}}, thanks!" } }, + ], ], - similes: [], - handler: function ( - _runtime: IAgentRuntime, - _message: Memory, - _state?: State, - _options?: { [key: string]: unknown }, - _callback?: HandlerCallback - ): Promise { - throw new Error("Function not implemented."); - }, - validate: function ( - _runtime: IAgentRuntime, - _message: Memory, - _state?: State - ): Promise { - throw new Error("Function not implemented."); - }, + similes: ["say hi", "welcome"], + handler: async () => { throw new Error("Not implemented"); }, + validate: async () => { throw new Error("Not implemented"); }, }, { name: "farewell", @@ -50,73 +34,123 @@ describe("Actions", () => { examples: [ [ { user: "user1", content: { text: "Goodbye {{user2}}!" } }, + { user: "user2", content: { text: "Bye {{user1}}!" } }, + ], + ], + similes: ["say bye", "leave"], + handler: async () => { throw new Error("Not implemented"); }, + validate: async () => { throw new Error("Not implemented"); }, + }, + { + name: "help", + description: "Get assistance", + examples: [ + [ + { user: "user1", content: { text: "Can you help me {{user2}}?" } }, { user: "user2", - content: { text: "See you later {{user1}}!" }, + content: { text: "Of course {{user1}}, what do you need?", action: "assist" } }, ], ], - similes: [], - handler: function ( - _runtime: IAgentRuntime, - _message: Memory, - _state?: State, - _options?: { [key: string]: unknown }, - _callback?: HandlerCallback - ): Promise { - throw new Error("Function not implemented."); - }, - validate: function ( - _runtime: IAgentRuntime, - _message: Memory, - _state?: State - ): Promise { - throw new Error("Function not implemented."); - }, + similes: ["assist", "support"], + handler: async () => { throw new Error("Not implemented"); }, + validate: async () => { throw new Error("Not implemented"); }, }, ]; describe("composeActionExamples", () => { - it("should generate the correct number of examples", () => { - const result = composeActionExamples(mockActions, 1); - const exampleLines = result - .split("\n") - .filter((line) => line.length > 0); - expect(exampleLines.length).toBe(2); // Each example has 2 messages + it("should generate examples with correct format", () => { + const examples = composeActionExamples(mockActions, 1); + const lines = examples.trim().split("\n"); + expect(lines.length).toBeGreaterThan(0); + expect(lines[0]).toMatch(/^user\d: .+/); + }); + + it("should replace user placeholders with generated names", () => { + const examples = composeActionExamples(mockActions, 1); + expect(examples).not.toContain("{{user1}}"); + expect(examples).not.toContain("{{user2}}"); + }); + + it("should handle empty actions array", () => { + const examples = composeActionExamples([], 5); + expect(examples).toBe(""); }); - it("should replace placeholder names with generated names", () => { - const result = composeActionExamples(mockActions, 1); - expect(result).not.toContain("{{user1}}"); - expect(result).not.toContain("{{user2}}"); + it("should handle count larger than available examples", () => { + const examples = composeActionExamples(mockActions, 10); + expect(examples.length).toBeGreaterThan(0); }); }); describe("formatActionNames", () => { it("should format action names correctly", () => { - const result = formatActionNames(mockActions); - const names = result.split(", ").sort(); - expect(names).toEqual(["farewell", "greet"].sort()); + const formatted = formatActionNames([mockActions[0], mockActions[1]]); + expect(formatted).toMatch(/^(greet|farewell)(, (greet|farewell))?$/); + }); + + it("should handle single action", () => { + const formatted = formatActionNames([mockActions[0]]); + expect(formatted).toBe("greet"); }); - it("should return empty string for empty array", () => { - const result = formatActionNames([]); - expect(result).toBe(""); + it("should handle empty actions array", () => { + const formatted = formatActionNames([]); + expect(formatted).toBe(""); }); }); describe("formatActions", () => { - it("should format actions with descriptions correctly", () => { - const result = formatActions(mockActions); - const formattedActions = result.split(",\n").sort(); - expect(formattedActions).toEqual( - ["farewell: Say goodbye", "greet: Greet someone"].sort() - ); + it("should format actions with descriptions", () => { + const formatted = formatActions([mockActions[0]]); + expect(formatted).toBe("greet: Greet someone"); + }); + + it("should include commas and newlines between multiple actions", () => { + const formatted = formatActions([mockActions[0], mockActions[1]]); + const parts = formatted.split(",\n"); + expect(parts.length).toBe(2); + expect(parts[0]).toMatch(/^(greet|farewell): /); + expect(parts[1]).toMatch(/^(greet|farewell): /); + }); + + it("should handle empty actions array", () => { + const formatted = formatActions([]); + expect(formatted).toBe(""); + }); + }); + + describe("Action Structure", () => { + it("should validate action structure", () => { + mockActions.forEach(action => { + expect(action).toHaveProperty("name"); + expect(action).toHaveProperty("description"); + expect(action).toHaveProperty("examples"); + expect(action).toHaveProperty("similes"); + expect(action).toHaveProperty("handler"); + expect(action).toHaveProperty("validate"); + expect(Array.isArray(action.examples)).toBe(true); + expect(Array.isArray(action.similes)).toBe(true); + }); + }); + + it("should validate example structure", () => { + mockActions.forEach(action => { + action.examples.forEach(example => { + example.forEach(message => { + expect(message).toHaveProperty("user"); + expect(message).toHaveProperty("content"); + expect(message.content).toHaveProperty("text"); + }); + }); + }); }); - it("should return empty string for empty array", () => { - const result = formatActions([]); - expect(result).toBe(""); + it("should have unique action names", () => { + const names = mockActions.map(action => action.name); + const uniqueNames = new Set(names); + expect(names.length).toBe(uniqueNames.size); }); }); }); diff --git a/packages/core/src/tests/messages.test.ts b/packages/core/src/tests/messages.test.ts index ce93e07db43..bbebe103a6d 100644 --- a/packages/core/src/tests/messages.test.ts +++ b/packages/core/src/tests/messages.test.ts @@ -23,7 +23,7 @@ describe("Messages Library", () => { } as unknown as IAgentRuntime; // Mock user data with proper UUID format - userId = "12345678-1234-1234-1234-123456789abc" as UUID; + userId = "123e4567-e89b-12d3-a456-426614174000" as UUID; actors = [ { id: userId, @@ -39,7 +39,7 @@ describe("Messages Library", () => { }); test("getActorDetails should return actors based on roomId", async () => { - const roomId: UUID = "room1234-1234-1234-1234-123456789abc" as UUID; + const roomId: UUID = "123e4567-e89b-12d3-a456-426614174001" as UUID; // Using vi.mocked() type assertion instead of jest.Mock casting vi.mocked( @@ -77,7 +77,7 @@ describe("Messages Library", () => { { content: { text: "Hello, world!" } as Content, userId: userId, - roomId: "room1234-1234-1234-1234-123456789abc" as UUID, + roomId: "123e4567-e89b-12d3-a456-426614174002" as UUID, createdAt: new Date().getTime(), agentId: "" as UUID, // assuming agentId is an empty string here }, @@ -105,14 +105,14 @@ describe("Messages Library", () => { text: "Check this attachment", attachments: [ { - id: "1", + id: "123e4567-e89b-12d3-a456-426614174003" as UUID, title: "Image", url: "http://example.com/image.jpg", }, ], } as Content, userId: userId, - roomId: "room1234-1234-1234-1234-123456789abc" as UUID, + roomId: "123e4567-e89b-12d3-a456-426614174004" as UUID, createdAt: new Date().getTime(), agentId: "" as UUID, // assuming agentId is an empty string here }, @@ -123,7 +123,7 @@ describe("Messages Library", () => { // Assertions expect(formattedMessages).toContain("Check this attachment"); expect(formattedMessages).toContain( - "Attachments: [1 - Image (http://example.com/image.jpg)]" + "Attachments: [" ); }); @@ -134,7 +134,7 @@ describe("Messages Library", () => { text: "No attachments here", } as Content, userId: userId, - roomId: "room1234-1234-1234-1234-123456789abc" as UUID, + roomId: "123e4567-e89b-12d3-a456-426614174005" as UUID, createdAt: new Date().getTime(), agentId: "" as UUID, // assuming agentId is an empty string here }, @@ -147,3 +147,187 @@ describe("Messages Library", () => { expect(formattedMessages).not.toContain("Attachments"); }); }); + +describe('Messages', () => { + const mockActors: Actor[] = [ + { + id: "123e4567-e89b-12d3-a456-426614174006" as UUID, + name: 'Alice', + username: 'alice', + details: { + tagline: 'Software Engineer', + summary: 'Full-stack developer with 5 years experience', + quote: "" + } + }, + { + id: "123e4567-e89b-12d3-a456-426614174007" as UUID, + name: 'Bob', + username: 'bob', + details: { + tagline: 'Product Manager', + summary: 'Experienced in agile methodologies', + quote: "" + } + } + ]; + + const mockMessages: Memory[] = [ + { + id: "123e4567-e89b-12d3-a456-426614174008" as UUID, + roomId: "123e4567-e89b-12d3-a456-426614174009" as UUID, + userId: mockActors[0].id, + createdAt: Date.now() - 5000, // 5 seconds ago + content: { + text: 'Hello everyone!', + action: 'wave' + } as Content, + agentId: "123e4567-e89b-12d3-a456-426614174001" + }, + { + id: "123e4567-e89b-12d3-a456-426614174010" as UUID, + roomId: "123e4567-e89b-12d3-a456-426614174009" as UUID, + userId: mockActors[1].id, + createdAt: Date.now() - 60000, // 1 minute ago + content: { + text: 'Hi Alice!', + attachments: [ + { + id: "123e4567-e89b-12d3-a456-426614174011" as UUID, + title: 'Document', + url: 'https://example.com/doc.pdf' + } + ] + } as Content, + agentId: "123e4567-e89b-12d3-a456-426614174001" + } + ]; + + describe('getActorDetails', () => { + it('should retrieve actor details from database', async () => { + const mockRuntime = { + databaseAdapter: { + getParticipantsForRoom: vi.fn().mockResolvedValue([mockActors[0].id, mockActors[1].id]), + getAccountById: vi.fn().mockImplementation((id) => { + const actor = mockActors.find(a => a.id === id); + return Promise.resolve(actor); + }) + } + }; + + const actors = await getActorDetails({ + runtime: mockRuntime as any, + roomId: "123e4567-e89b-12d3-a456-426614174009" as UUID + }); + + expect(actors).toHaveLength(2); + expect(actors[0].name).toBe('Alice'); + expect(actors[1].name).toBe('Bob'); + expect(mockRuntime.databaseAdapter.getParticipantsForRoom).toHaveBeenCalled(); + }); + + it('should filter out null actors', async () => { + const invalidId = "123e4567-e89b-12d3-a456-426614174012" as UUID; + const mockRuntime = { + databaseAdapter: { + getParticipantsForRoom: vi.fn().mockResolvedValue([mockActors[0].id, invalidId]), + getAccountById: vi.fn().mockImplementation((id) => { + const actor = mockActors.find(a => a.id === id); + return Promise.resolve(actor || null); + }) + } + }; + + const actors = await getActorDetails({ + runtime: mockRuntime as any, + roomId: "123e4567-e89b-12d3-a456-426614174009" as UUID + }); + + expect(actors).toHaveLength(1); + expect(actors[0].name).toBe('Alice'); + }); + }); + + describe('formatActors', () => { + it('should format actors with complete details', () => { + const formatted = formatActors({ actors: mockActors }); + expect(formatted).toContain('Alice: Software Engineer'); + expect(formatted).toContain('Full-stack developer with 5 years experience'); + expect(formatted).toContain('Bob: Product Manager'); + expect(formatted).toContain('Experienced in agile methodologies'); + }); + + it('should handle actors without details', () => { + const actorsWithoutDetails: Actor[] = [ + { + id: "123e4567-e89b-12d3-a456-426614174013" as UUID, + name: 'Charlie', + username: 'charlie', + details: { + tagline: "Tag", + summary: "Summary", + quote: "Quote" + } + } + ]; + const formatted = formatActors({ actors: actorsWithoutDetails }); + expect(formatted).toBe('Charlie: Tag\nSummary'); + }); + + it('should handle empty actors array', () => { + const formatted = formatActors({ actors: [] }); + expect(formatted).toBe(''); + }); + }); + + describe('formatMessages', () => { + it('should format messages with all details', () => { + const formatted = formatMessages({ messages: mockMessages, actors: mockActors }); + const lines = formatted.split('\n'); + expect(lines[1]).toContain("Alice"); + expect(lines[1]).toContain("(wave)"); + expect(lines[1]).toContain("(just now)"); + }); + + it('should handle messages from unknown users', () => { + const messagesWithUnknownUser: Memory[] = [{ + id: "123e4567-e89b-12d3-a456-426614174014" as UUID, + roomId: "123e4567-e89b-12d3-a456-426614174009" as UUID, + userId: "123e4567-e89b-12d3-a456-426614174015" as UUID, + createdAt: Date.now(), + content: { text: 'Test message' } as Content, + agentId: "123e4567-e89b-12d3-a456-426614174001" + }]; + + const formatted = formatMessages({ messages: messagesWithUnknownUser, actors: mockActors }); + expect(formatted).toContain('Unknown User: Test message'); + }); + + it('should handle messages with no action', () => { + const messagesWithoutAction: Memory[] = [{ + id: "123e4567-e89b-12d3-a456-426614174016" as UUID, + roomId: "123e4567-e89b-12d3-a456-426614174009" as UUID, + userId: mockActors[0].id, + createdAt: Date.now(), + content: { text: 'Simple message' } as Content, + agentId: "123e4567-e89b-12d3-a456-426614174001" + }]; + + const formatted = formatMessages({ messages: messagesWithoutAction, actors: mockActors }); + expect(formatted).not.toContain('()'); + expect(formatted).toContain('Simple message'); + }); + + it('should handle empty messages array', () => { + const formatted = formatMessages({ messages: [], actors: mockActors }); + expect(formatted).toBe(''); + }); + }); + + describe('formatTimestamp', () => { + it('should handle exact time boundaries', () => { + const now = Date.now(); + expect(formatTimestamp(now)).toContain('just now'); + }); + }); +}); diff --git a/packages/core/src/tests/models.test.ts b/packages/core/src/tests/models.test.ts index 249dd587e69..8fd4aa8495b 100644 --- a/packages/core/src/tests/models.test.ts +++ b/packages/core/src/tests/models.test.ts @@ -1,54 +1,146 @@ -import { getModel, getEndpoint } from "../models.ts"; +import { getModel, getEndpoint, models } from "../models.ts"; import { ModelProviderName, ModelClass } from "../types.ts"; import { describe, test, expect, vi } from "vitest"; +// Mock settings vi.mock("../settings", () => { return { default: { SMALL_OPENROUTER_MODEL: "mock-small-model", + LARGE_OPENROUTER_MODEL: "mock-large-model", OPENROUTER_MODEL: "mock-default-model", OPENAI_API_KEY: "mock-openai-key", ANTHROPIC_API_KEY: "mock-anthropic-key", OPENROUTER_API_KEY: "mock-openrouter-key", + ETERNALAI_MODEL: "mock-eternal-model", + ETERNALAI_URL: "https://mock.eternal.ai", + LLAMACLOUD_MODEL_SMALL: "mock-llama-small", + LLAMACLOUD_MODEL_LARGE: "mock-llama-large", + TOGETHER_MODEL_SMALL: "mock-together-small", + TOGETHER_MODEL_LARGE: "mock-together-large", }, loadEnv: vi.fn(), }; }); -describe("Model Provider Tests", () => { - test("should retrieve the correct model for OpenAI SMALL", () => { - const model = getModel(ModelProviderName.OPENAI, ModelClass.SMALL); - expect(model).toBe("gpt-4o-mini"); +describe("Model Provider Configuration", () => { + describe("OpenAI Provider", () => { + test("should have correct endpoint", () => { + expect(models[ModelProviderName.OPENAI].endpoint).toBe("https://api.openai.com/v1"); + }); + + test("should have correct model mappings", () => { + const openAIModels = models[ModelProviderName.OPENAI].model; + expect(openAIModels[ModelClass.SMALL]).toBe("gpt-4o-mini"); + expect(openAIModels[ModelClass.MEDIUM]).toBe("gpt-4o"); + expect(openAIModels[ModelClass.LARGE]).toBe("gpt-4o"); + expect(openAIModels[ModelClass.EMBEDDING]).toBe("text-embedding-3-small"); + expect(openAIModels[ModelClass.IMAGE]).toBe("dall-e-3"); + }); + + test("should have correct settings configuration", () => { + const settings = models[ModelProviderName.OPENAI].settings; + expect(settings.maxInputTokens).toBe(128000); + expect(settings.maxOutputTokens).toBe(8192); + expect(settings.temperature).toBe(0.6); + expect(settings.frequency_penalty).toBe(0.0); + expect(settings.presence_penalty).toBe(0.0); + }); + }); + + describe("Anthropic Provider", () => { + test("should have correct endpoint", () => { + expect(models[ModelProviderName.ANTHROPIC].endpoint).toBe("https://api.anthropic.com/v1"); + }); + + test("should have correct model mappings", () => { + const anthropicModels = models[ModelProviderName.ANTHROPIC].model; + expect(anthropicModels[ModelClass.SMALL]).toBe("claude-3-haiku-20240307"); + expect(anthropicModels[ModelClass.MEDIUM]).toBe("claude-3-5-sonnet-20241022"); + expect(anthropicModels[ModelClass.LARGE]).toBe("claude-3-5-sonnet-20241022"); + }); + + test("should have correct token limits", () => { + const settings = models[ModelProviderName.ANTHROPIC].settings; + expect(settings.maxInputTokens).toBe(200000); + expect(settings.maxOutputTokens).toBe(4096); + }); + }); + + describe("Google Provider", () => { + test("should have correct model mappings", () => { + const googleModels = models[ModelProviderName.GOOGLE].model; + expect(googleModels[ModelClass.SMALL]).toBe("gemini-1.5-flash-latest"); + expect(googleModels[ModelClass.MEDIUM]).toBe("gemini-1.5-flash-latest"); + expect(googleModels[ModelClass.LARGE]).toBe("gemini-1.5-pro-latest"); + }); }); +}); + +describe("Model Retrieval Functions", () => { + describe("getModel function", () => { + test("should retrieve correct models for all providers", () => { + expect(getModel(ModelProviderName.OPENAI, ModelClass.SMALL)).toBe("gpt-4o-mini"); + expect(getModel(ModelProviderName.ANTHROPIC, ModelClass.LARGE)).toBe("claude-3-5-sonnet-20241022"); + expect(getModel(ModelProviderName.GOOGLE, ModelClass.MEDIUM)).toBe("gemini-1.5-flash-latest"); + }); + + test("should handle environment variable overrides", () => { + expect(getModel(ModelProviderName.OPENROUTER, ModelClass.SMALL)).toBe("mock-small-model"); + expect(getModel(ModelProviderName.OPENROUTER, ModelClass.LARGE)).toBe("mock-large-model"); + expect(getModel(ModelProviderName.ETERNALAI, ModelClass.SMALL)).toBe("mock-eternal-model"); + }); - test("should retrieve the correct model for Google MEDIUM", () => { - const model = getModel(ModelProviderName.GOOGLE, ModelClass.MEDIUM); - expect(model).toBe("gemini-1.5-flash-latest"); + test("should throw error for invalid model provider", () => { + expect(() => getModel("INVALID_PROVIDER" as any, ModelClass.SMALL)).toThrow(); + }); }); - test("should retrieve the correct model for Groq LARGE", () => { - const model = getModel(ModelProviderName.GROQ, ModelClass.LARGE); - expect(model).toBe("llama-3.2-90b-vision-preview"); + describe("getEndpoint function", () => { + test("should retrieve correct endpoints for all providers", () => { + expect(getEndpoint(ModelProviderName.OPENAI)).toBe("https://api.openai.com/v1"); + expect(getEndpoint(ModelProviderName.ANTHROPIC)).toBe("https://api.anthropic.com/v1"); + expect(getEndpoint(ModelProviderName.ETERNALAI)).toBe("https://mock.eternal.ai"); + }); + + test("should throw error for invalid provider", () => { + expect(() => getEndpoint("INVALID_PROVIDER" as any)).toThrow(); + }); }); +}); - test("should retrieve the correct model for OpenRouter SMALL", () => { - const model = getModel(ModelProviderName.OPENROUTER, ModelClass.SMALL); - expect(model).toBe("mock-small-model"); +describe("Model Settings Validation", () => { + test("all providers should have required settings", () => { + Object.values(ModelProviderName).forEach(provider => { + const providerConfig = models[provider]; + expect(providerConfig.settings).toBeDefined(); + expect(providerConfig.settings.maxInputTokens).toBeGreaterThan(0); + expect(providerConfig.settings.maxOutputTokens).toBeGreaterThan(0); + expect(providerConfig.settings.temperature).toBeDefined(); + }); }); - test("should retrieve the correct endpoint for OpenAI", () => { - const endpoint = getEndpoint(ModelProviderName.OPENAI); - expect(endpoint).toBe("https://api.openai.com/v1"); + test("all providers should have model mappings for basic model classes", () => { + Object.values(ModelProviderName).forEach(provider => { + const providerConfig = models[provider]; + expect(providerConfig.model).toBeDefined(); + expect(providerConfig.model[ModelClass.SMALL]).toBeDefined(); + expect(providerConfig.model[ModelClass.MEDIUM]).toBeDefined(); + expect(providerConfig.model[ModelClass.LARGE]).toBeDefined(); + }); }); +}); - test("should retrieve the correct endpoint for Anthropic", () => { - const endpoint = getEndpoint(ModelProviderName.ANTHROPIC); - expect(endpoint).toBe("https://api.anthropic.com/v1"); +describe("Environment Variable Integration", () => { + test("should use environment variables for LlamaCloud models", () => { + const llamaConfig = models[ModelProviderName.LLAMACLOUD]; + expect(llamaConfig.model[ModelClass.SMALL]).toBe("meta-llama/Llama-3.2-3B-Instruct-Turbo"); + expect(llamaConfig.model[ModelClass.LARGE]).toBe("meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo"); }); - test("should handle invalid model provider for getModel", () => { - expect(() => - getModel("INVALID_PROVIDER" as any, ModelClass.SMALL) - ).toThrow(); + test("should use environment variables for Together models", () => { + const togetherConfig = models[ModelProviderName.TOGETHER]; + expect(togetherConfig.model[ModelClass.SMALL]).toBe("meta-llama/Llama-3.2-3B-Instruct-Turbo"); + expect(togetherConfig.model[ModelClass.LARGE]).toBe("meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo"); }); });