fix/webhook history pagination

apps/web/app/api/google/webhook/process-history.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, expect, vi, beforeEach } from "vitest";
 import { processHistoryForUser } from "./process-history";
 import { getHistory } from "@/utils/gmail/history";
+import { processHistoryItem } from "@/app/api/google/webhook/process-history-item";
 import {
   getWebhookEmailAccount,
   validateWebhookAccount,
@@ -22,6 +23,10 @@ vi.mock("@/utils/gmail/history", () => ({
   getHistory: vi.fn(),
 }));
 
+vi.mock("@/app/api/google/webhook/process-history-item", () => ({
+  processHistoryItem: vi.fn(),
+}));
+
 vi.mock("@/utils/webhook/validate-webhook-account", () => ({
   getWebhookEmailAccount: vi.fn(),
   validateWebhookAccount: vi.fn(),
@@ -132,4 +137,145 @@ describe("processHistoryForUser - 404 Handling", () => {
       }),
     );
   });
+
+  it("should aggregate history across pages", async () => {
+    const email = "user@test.com";
+    const historyId = 1200;
+    const emailAccount = {
+      id: "account-123",
+      email,
+      lastSyncedHistoryId: "1000",
+    };
+
+    vi.mocked(getWebhookEmailAccount).mockResolvedValue(emailAccount as any);
+    vi.mocked(validateWebhookAccount).mockResolvedValue({
+      success: true,
+      data: {
+        emailAccount: {
+          ...emailAccount,
+          account: {
+            access_token: "token",
+            refresh_token: "refresh",
+            expires_at: new Date(Date.now() + 3_600_000),
+          },
+          rules: [],
+        },
+        hasAutomationRules: false,
+        hasAiAccess: false,
+      },
+    } as any);
+
+    vi.mocked(getHistory)
+      .mockResolvedValueOnce({
+        history: [
+          {
+            id: "1101",
+            messagesAdded: [
+              {
+                message: {
+                  id: "m-1",
+                  threadId: "t-1",
+                  labelIds: ["INBOX"],
+                },
+              },
+            ],
+          },
+        ],
+        nextPageToken: "page-2",
+      } as any)
+      .mockResolvedValueOnce({
+        history: [
+          {
+            id: "1102",
+            messagesAdded: [
+              {
+                message: {
+                  id: "m-2",
+                  threadId: "t-2",
+                  labelIds: ["INBOX"],
+                },
+              },
+            ],
+          },
+        ],
+      } as any);
+
+    await processHistoryForUser({ emailAddress: email, historyId }, {}, logger);
+
+    expect(vi.mocked(getHistory)).toHaveBeenCalledTimes(2);
+    expect(vi.mocked(processHistoryItem)).toHaveBeenCalledTimes(2);
+    expect(vi.mocked(processHistoryItem)).toHaveBeenCalledWith(
+      expect.objectContaining({
+        item: expect.objectContaining({
+          message: expect.objectContaining({ id: "m-1" }),
+        }),
+      }),
+      expect.any(Object),
+      expect.any(Object),
+    );
+    expect(vi.mocked(processHistoryItem)).toHaveBeenCalledWith(
+      expect.objectContaining({
+        item: expect.objectContaining({
+          message: expect.objectContaining({ id: "m-2" }),
+        }),
+      }),
+      expect.any(Object),
+      expect.any(Object),
+    );
+  });
+
+  it("should warn when Gmail history pagination is capped", async () => {
+    const email = "user@test.com";
+    const historyId = 1200;
+    const emailAccount = {
+      id: "account-123",
+      email,
+      lastSyncedHistoryId: "1000",
+    };
+
+    vi.mocked(getWebhookEmailAccount).mockResolvedValue(emailAccount as any);
+    vi.mocked(validateWebhookAccount).mockResolvedValue({
+      success: true,
+      data: {
+        emailAccount: {
+          ...emailAccount,
+          account: {
+            access_token: "token",
+            refresh_token: "refresh",
+            expires_at: new Date(Date.now() + 3_600_000),
+          },
+          rules: [],
+        },
+        hasAutomationRules: false,
+        hasAiAccess: false,
+      },
+    } as any);
+
+    vi.mocked(getHistory)
+      .mockResolvedValueOnce({
+        history: [],
+        nextPageToken: "page-2",
+      } as any)
+      .mockResolvedValueOnce({
+        history: [],
+        nextPageToken: "page-3",
+      } as any)
+      .mockResolvedValueOnce({
+        history: [],
+        nextPageToken: "page-4",
+      } as any);
+
+    const warnSpy = vi.spyOn(logger, "warn");
+
+    await processHistoryForUser({ emailAddress: email, historyId }, {}, logger);
+
+    expect(vi.mocked(getHistory)).toHaveBeenCalledTimes(3);
+    expect(warnSpy).toHaveBeenCalledWith(
+      "Gmail history pagination capped",
+      expect.objectContaining({
+        pagesFetched: 3,
+        maxPages: 3,
+      }),
+    );
+  });
 });

apps/web/app/api/google/webhook/process-history.ts

@@ -335,16 +335,50 @@ async function fetchGmailHistoryResilient({
   });
 
   try {
-    const data = await getHistory(gmail, {
-      startHistoryId,
-      historyTypes: ["messageAdded", "labelAdded", "labelRemoved"],
-      maxResults: 500,
-    });
+    const maxPages = 3;
+    const maxHistoryItems = 1500;
+    let pageToken: string | undefined;
+    let pagesFetched = 0;
+    let capped = false;
+    let lastResponse: Awaited<ReturnType<typeof getHistory>> | null = null;
+    let historyItems: gmail_v1.Schema$History[] = [];
+
+    do {
+      const data = await getHistory(gmail, {
+        startHistoryId,
+        historyTypes: ["messageAdded", "labelAdded", "labelRemoved"],
+        maxResults: 500,
+        pageToken,
+      });
+      lastResponse = data;
+      if (data.history?.length) {
+        historyItems = historyItems.concat(data.history);
+      }
+      pagesFetched += 1;
+      pageToken = data.nextPageToken || undefined;
+
+      if (
+        pageToken &&
+        (pagesFetched >= maxPages || historyItems.length >= maxHistoryItems)
+      ) {
+        capped = true;
+        break;
+      }
+    } while (pageToken);
+
+    const data: Awaited<ReturnType<typeof getHistory>> = {
+      ...lastResponse,
+      history: historyItems,
+      nextPageToken: capped ? lastResponse.nextPageToken : undefined,
+    };
 
     if (data.nextPageToken) {
-      logger.warn("Gmail history has more pages that were not fetched", {
+      logger.warn("Gmail history pagination capped", {
         historyItemCount: data.history?.length ?? 0,
         startHistoryId,
+        pagesFetched,
+        maxPages,
+        maxHistoryItems,
       });
     }
 

apps/web/app/api/google/webhook/route.ts

@@ -6,7 +6,7 @@ import type { Logger } from "@/utils/logger";
 import { handleWebhookError } from "@/utils/webhook/error-handler";
 import { getWebhookEmailAccount } from "@/utils/webhook/validate-webhook-account";
 
-export const maxDuration = 300;
+export const maxDuration = 800;
 
 // Google PubSub calls this endpoint each time a user recieves an email. We subscribe for updates via `api/google/watch`
 export const POST = withError("google/webhook", async (request) => {

apps/web/app/api/outlook/webhook/route.ts

@@ -8,7 +8,7 @@ import { webhookBodySchema } from "@/app/api/outlook/webhook/types";
 import { handleWebhookError } from "@/utils/webhook/error-handler";
 import { getWebhookEmailAccount } from "@/utils/webhook/validate-webhook-account";
 
-export const maxDuration = 300;
+export const maxDuration = 800;
 
 export const POST = withError("outlook/webhook", async (request) => {
   const searchParams = new URL(request.url).searchParams;

apps/web/utils/ai/digest/summarize-email-for-digest.ts

@@ -68,7 +68,8 @@ Guidelines for summarizing the email:
   • Use newlines if there are multiple action items or pieces of information
 - Only include human-relevant and human-readable information.
 - Exclude opaque technical identifiers like account IDs, payment IDs, tracking tokens, or long alphanumeric strings that aren't meaningful to users.
-`;
+
+Return your response in JSON format.`;
 
   const prompt = `
 <email>

apps/web/utils/ai/document-filing/parse-filing-reply.ts

@@ -13,7 +13,9 @@ Actions:
 - "undo": User wants to reverse the filing. We will move the file to a "To Delete" folder for them to review.
 - "none": No action needed, just answering a question or continuing conversation.
 
-Always write a helpful, concise reply.`;
+Always write a helpful, concise reply.
+
+Return your response in JSON format.`;
 
 const schema = z.object({
   action: z.enum(["approve", "move", "undo", "none"]),

apps/web/utils/ai/report/analyze-email-behavior.ts

@@ -33,7 +33,9 @@ export async function aiAnalyzeEmailBehavior(
 ) {
   const system = `You are an expert AI system that analyzes a user's email behavior to infer timing patterns, content preferences, and automation opportunities.
 
-Focus on identifying patterns that can be automated and providing specific, actionable automation rules that would save time and improve email management efficiency.`;
+Focus on identifying patterns that can be automated and providing specific, actionable automation rules that would save time and improve email management efficiency.
+
+Return your response in JSON format.`;
 
   const prompt = `### Email Analysis Data
 

apps/web/utils/ai/report/analyze-label-optimization.ts

@@ -28,7 +28,9 @@ export async function aiAnalyzeLabelOptimization(
 ): Promise<z.infer<typeof labelAnalysisSchema>> {
   const system = `You are a Gmail organization expert. Analyze the user's current labels and email patterns to suggest specific optimizations that will improve their email organization and workflow efficiency.
 
-Focus on practical suggestions that will reduce email management time and improve organization.`;
+Focus on practical suggestions that will reduce email management time and improve organization.
+
+Return your response in JSON format.`;
 
   const prompt = `### Current Gmail Labels
 ${gmailLabels.map((label) => `- ${label.name}: ${label.messagesTotal || 0} emails, ${label.messagesUnread || 0} unread`).join("\n")}

apps/web/utils/ai/report/build-user-persona.ts

@@ -30,7 +30,9 @@ Analyze the email summaries, signatures, and templates to identify:
 1. Professional identity with supporting evidence
 2. Current professional priorities based on email content
 
-Focus on understanding the user's role and what they're currently focused on professionally.`;
+Focus on understanding the user's role and what they're currently focused on professionally.
+
+Return your response in JSON format.`;
 
   const prompt = `### Input Data
 

apps/web/utils/ai/report/generate-actionable-recommendations.ts

@@ -42,7 +42,9 @@ export async function aiGenerateActionableRecommendations(
 ): Promise<z.infer<typeof actionableRecommendationsSchema>> {
   const system = `You are an email productivity consultant. Based on the comprehensive email analysis, create specific, actionable recommendations that the user can implement to improve their email workflow.
 
-Organize recommendations by timeline (immediate, short-term, long-term) and include specific implementation details and expected benefits.`;
+Organize recommendations by timeline (immediate, short-term, long-term) and include specific implementation details and expected benefits.
+
+Return your response in JSON format.`;
 
   const prompt = `### Analysis Summary
 

apps/web/utils/ai/report/generate-executive-summary.ts

@@ -79,7 +79,9 @@ Examples of BAD personas (too vague):
 - "Tech Worker"
 - "Knowledge Worker"
 
-Focus on identifying the PRIMARY professional role based on email content, senders, and communication patterns.`;
+Focus on identifying the PRIMARY professional role based on email content, senders, and communication patterns.
+
+Return your response in JSON format.`;
 
   const prompt = `### Email Analysis Data
 

apps/web/utils/ai/report/response-patterns.ts

@@ -58,7 +58,9 @@ IMPORTANT: When creating email categories, avoid meaningless or generic categori
 - "Unclear Content/HTML Code", "HTML Content", "Raw Content"
 - "General", "Random", "Various"
 
-Only suggest categories that are meaningful and provide clear organizational value. If an email doesn't fit into a meaningful category, don't create a category for it.`;
+Only suggest categories that are meaningful and provide clear organizational value. If an email doesn't fit into a meaningful category, don't create a category for it.
+
+Return your response in JSON format.`;
 
   const prompt = `### Input Data
 

apps/web/utils/gmail/history.ts

@@ -7,6 +7,7 @@ export async function getHistory(
     startHistoryId: string;
     historyTypes?: string[];
     maxResults?: number;
+    pageToken?: string;
   },
 ) {
   const history = await withGmailRetry(() =>
@@ -15,6 +16,7 @@ export async function getHistory(
       startHistoryId: options.startHistoryId,
       historyTypes: options.historyTypes,
       maxResults: options.maxResults,
+      pageToken: options.pageToken,
     }),
   );