refactored debug log for ai-collab

packages/framework/ai-collab/README.md

@@ -85,7 +85,7 @@ export class PlannerAppState extends sf.object("PlannerAppState", {
 ### Example 1: Collaborate with AI
 
 ```ts
-import { aiCollab } from "@fluidframework/ai-collab/alpha";
+import { aiCollab, DebugEvent } from "@fluidframework/ai-collab/alpha";
 import { PlannerAppState } from "./types.ts"
 // This is not a real file, this is meant to represent how you initialize your app data.
 import { initializeAppState } from "./yourAppInitializationFile.ts"
@@ -145,7 +145,7 @@ const response = await aiCollab({
 		},
 		planningStep: true,
 		finalReviewStep: true,
-		dumpDebugLog: true,
+		debugEventLogHandler: (event: DebugEvent) => {console.log(event);}
 	});
 
 if (response.status === 'sucess') {

packages/framework/ai-collab/api-report/ai-collab.alpha.api.md

@@ -16,7 +16,7 @@ export interface AiCollabErrorResponse {
 
 // @alpha
 export interface AiCollabOptions {
-    readonly dumpDebugLog?: boolean;
+    readonly debugEventLogHandler?: DebugEventLogHandler;
     readonly finalReviewStep?: boolean;
     readonly limiters?: {
         readonly abortController?: AbortController;
@@ -46,6 +46,21 @@ export function createMergableDiffSeries(diffs: Difference[]): Difference[];
 // @alpha
 export function createMergableIdDiffSeries(oldObject: unknown, diffs: Difference[], idAttributeName: string | number): Difference[];
 
+// @alpha
+export interface DebugEvent {
+    // (undocumented)
+    eventName?: string;
+    // (undocumented)
+    id: string;
+    // (undocumented)
+    timestamp: string;
+    // (undocumented)
+    traceId?: string;
+}
+
+// @alpha
+export type DebugEventLogHandler = <T extends DebugEvent>(event: T) => unknown;
+
 // @alpha
 export type Difference = DifferenceCreate | DifferenceRemove | DifferenceChange | DifferenceMove;
 

packages/framework/ai-collab/package.json

@@ -114,8 +114,10 @@
 		"@fluidframework/runtime-utils": "workspace:~",
 		"@fluidframework/telemetry-utils": "workspace:~",
 		"@fluidframework/tree": "workspace:~",
+		"@types/uuid": "^9.0.2",
 		"openai": "^4.67.3",
 		"typechat": "^0.1.1",
+		"uuid": "^9.0.0",
 		"zod": "^3.23.8"
 	},
 	"devDependencies": {
@@ -142,7 +144,8 @@
 		"mocha-multi-reporters": "^1.5.1",
 		"prettier": "~3.0.3",
 		"rimraf": "^4.4.0",
-		"typescript": "~5.4.5"
+		"typescript": "~5.4.5",
+		"uuid": "^9.0.0"
 	},
 	"fluidBuild": {
 		"tasks": {

packages/framework/ai-collab/src/aiCollab.ts

@@ -53,7 +53,6 @@ import { generateTreeEdits } from "./explicit-strategy/index.js";
  * 		},
  * 		planningStep: true,
  * 		finalReviewStep: true,
- * 		dumpDebugLog: true,
  * 	});
  * ```
  *
@@ -77,9 +76,9 @@ export async function aiCollab(
 		openAI: options.openAI,
 		prompt: options.prompt,
 		limiters: options.limiters,
-		dumpDebugLog: options.dumpDebugLog,
 		planningStep: options.planningStep,
 		finalReviewStep: options.finalReviewStep,
+		debugEventLogHandler: options.debugEventLogHandler,
 	});
 
 	return response;

packages/framework/ai-collab/src/aiCollabApi.ts

@@ -7,6 +7,24 @@ import type { TreeNode } from "@fluidframework/tree";
 // eslint-disable-next-line import/no-named-as-default
 import type OpenAI from "openai";
 
+/**
+ * Debug event types for the ai-collab
+ *
+ * @alpha
+ */
+export interface DebugEvent {
+	id: string;
+	traceId?: string;
+	eventName?: string;
+	timestamp: string;
+}
+
+/**
+ * A callback function that can be used to handle debug events that occur during the AI collaboration process.
+ * @alpha
+ */
+export type DebugEventLogHandler = <T extends DebugEvent>(event: T) => unknown;
+
 /**
  * OpenAI client options for the {@link AiCollabOptions} interface.
  *
@@ -99,9 +117,9 @@ export interface AiCollabOptions {
 	 */
 	readonly validator?: (newContent: TreeNode) => void;
 	/**
-	 * When enabled, the library will console.log information useful for debugging the AI collaboration.
+	 * An optional handler for debug events that occur during the AI collaboration.
 	 */
-	readonly dumpDebugLog?: boolean;
+	readonly debugEventLogHandler?: DebugEventLogHandler;
 }
 
 /**

packages/framework/ai-collab/src/explicit-strategy/debugEventLogTypes.ts

@@ -0,0 +1,121 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import type { DebugEvent } from "../aiCollabApi.js";
+
+import type { TreeEdit } from "./agentEditTypes.js";
+
+// Planning Prompt Debug events: ------------------------------------------------------
+
+/**
+ * @alpha
+ */
+export interface PlanningPromptInitiatedDebugEvent extends DebugEvent {
+	eventName: "GENERATE_PLANNING_PROMPT_LLM";
+	eventFlowStatus: "INITIATED";
+}
+
+/**
+ * @alpha
+ */
+export interface PlanningPromptCompletedDebugEvent extends DebugEvent {
+	eventName: "GENERATE_PLANNING_PROMPT_LLM";
+	eventFlowStatus: "COMPLETED";
+	requestOutcome: "success" | "failure";
+	llmGeneratedPlan: string | undefined;
+}
+
+// Editing System prompt Debug events: ------------------------------------------------
+
+/**
+ * A debug event marking the initiation of the flow for prompting an LLM to generate an edit to a SharedTree.
+ * @alpha
+ */
+export interface GenerateTreeEditInitiatedDebugEvent extends DebugEvent {
+	eventName: "GENERATE_TREE_EDIT_LLM";
+	eventFlowStatus: "INITIATED";
+}
+
+/**
+ * A debug event marking the completion of the flow for prompting an LLM to generate an edit to a SharedTree.
+ * @alpha
+ */
+export interface GenerateTreeEditCompletedDebugEvent extends DebugEvent {
+	eventName: "GENERATE_TREE_EDIT_LLM";
+	eventFlowStatus: "COMPLETED";
+	requestOutcome: "success" | "failure";
+	/**
+	 * This will be null if the LLM decides no more edits are necessary.
+	 */
+	// eslint-disable-next-line @rushstack/no-new-null
+	llmGeneratedEdit?: TreeEdit | null;
+}
+
+// Apply Edit Debug events: ----------------------------------------------------------
+
+/**
+ * A debug event marking the successful application of an LLM generated edit to a SharedTree.
+ * @alpha
+ */
+export interface ApplyEditSuccessDebugEvent extends DebugEvent {
+	eventName: "APPLIED_EDIT_SUCCESS";
+	edit: TreeEdit;
+}
+
+/**
+ * A debug event marking the failure of applying an LLM generated edit to a SharedTree.
+ * @alpha
+ */
+export interface ApplyEditFailureDebugEvent extends DebugEvent {
+	eventName: "APPLIED_EDIT_FAILURE";
+	edit: TreeEdit;
+	errorMessage: string;
+	sequentialErrorCount: number;
+}
+
+// Generate Final Review Debug events: ----------------------------------------------------------
+
+/**
+ * A debug event marking the initiation of the flow for prompting an LLM to complete a final review of its edits
+ * and determine whether the user's goal was accomplished.
+ * @alpha
+ */
+export interface FinalReviewInitiatedDebugEvent extends DebugEvent {
+	eventName: "FINAL_REVIEW_LLM";
+	eventFlowStatus: "INITIATED";
+	prompt: string;
+}
+
+/**
+ * A debug event marking the end of the flow for prompting an LLM to complete a final review of its edits
+ * and determine whether the user's goal was accomplished.
+ * @alpha
+ */
+export interface FinalReviewCompletedDebugEvent extends DebugEvent {
+	eventName: "FINAL_REVIEW_LLM";
+	eventFlowStatus: "COMPLETED";
+	prompt: string;
+	status: "success" | "failure";
+	llmReviewResponse?: {
+		goalAccomplished: "yes" | "no";
+	};
+}
+
+// Raw LLM Request/Response Debug Events ----------------------------------------------------------
+
+/**
+ * A debug event for an API call directly to a LLM.
+ * @alpha
+ */
+export interface LlmApiCallDebugEvent extends DebugEvent {
+	eventName: "LLM_API_CALL";
+	modelName: string;
+	requestParams: unknown;
+	response: unknown;
+	tokenUsage?: {
+		promptTokens: number;
+		completionTokens: number;
+	};
+}