From 5fc762c491404af95c8d2fdfd9be7c306e403c7b Mon Sep 17 00:00:00 2001 From: fegloff Date: Fri, 1 Nov 2024 11:10:35 -0500 Subject: [PATCH 1/2] add sliding window logic + add timestamp field --- src/modules/llms/api/athropic.ts | 15 ++++--- src/modules/llms/api/llmApi.ts | 7 ++-- src/modules/llms/api/openai.ts | 15 ++++--- src/modules/llms/api/pdfHandler.ts | 3 +- src/modules/llms/api/vertex.ts | 10 +++-- src/modules/llms/llmsBase.ts | 18 +++++--- src/modules/llms/utils/conversationManager.ts | 41 +++++++++++++++++++ src/modules/subagents/llamaSubagent.ts | 5 ++- src/modules/types.ts | 3 ++ src/modules/voice-to-voice-gpt/index.ts | 2 +- 10 files changed, 93 insertions(+), 26 deletions(-) create mode 100644 src/modules/llms/utils/conversationManager.ts diff --git a/src/modules/llms/api/athropic.ts b/src/modules/llms/api/athropic.ts index b2fba812..35686844 100644 --- a/src/modules/llms/api/athropic.ts +++ b/src/modules/llms/api/athropic.ts @@ -49,7 +49,8 @@ export const anthropicCompletion = async ( completion: { content: completion[0].text, role: 'assistant', - model + model, + timestamp: Date.now() }, usage: totalOutputTokens + totalInputTokens, price: 0, @@ -92,7 +93,8 @@ export const xaiCompletion = async ( completion: { content: completion[0].text, role: 'assistant', - model + model, + timestamp: Date.now() }, usage: totalOutputTokens + totalInputTokens, price: 0, @@ -202,7 +204,8 @@ export const anthropicStreamCompletion = async ( completion: { content: completion, role: 'assistant', - model + model, + timestamp: Date.now() }, usage: parseInt(totalOutputTokens, 10) + parseInt(totalInputTokens, 10), price: 0, @@ -252,7 +255,8 @@ export const toolsChatCompletion = async ( completion: { content: completion[0].text, role: 'assistant', - model + model, + timestamp: Date.now() }, usage: totalOutputTokens + totalInputTokens, price: 0, @@ -264,7 +268,8 @@ export const toolsChatCompletion = async ( completion: { content: 'Timeout error', role: 'assistant', - model + model, + timestamp: Date.now() }, usage: 0, price: 0 diff --git a/src/modules/llms/api/llmApi.ts b/src/modules/llms/api/llmApi.ts index f8ed9e8a..d29cefcc 100644 --- a/src/modules/llms/api/llmApi.ts +++ b/src/modules/llms/api/llmApi.ts @@ -1,6 +1,6 @@ import axios from 'axios' import config from '../../../config' -import { type ChatConversation } from '../../types' +import { type ChatConversationWithoutTimestamp, type ChatConversation } from '../../types' import pino from 'pino' import { type ChatModel } from '../utils/types' import { headers } from './helper' @@ -36,7 +36,7 @@ interface LlmAddUrlDocument { interface QueryUrlDocument { collectioName: string prompt: string - conversation?: ChatConversation[] + conversation?: ChatConversationWithoutTimestamp[] } export const getChatModel = (modelName: string): ChatModel | undefined => { @@ -130,7 +130,8 @@ export const llmCompletion = async ( completion: { content: completion[0].message?.content, role: 'system', - model + model, + timestamp: Date.now() }, usage: totalOutputTokens + totalInputTokens, price: 0 diff --git a/src/modules/llms/api/openai.ts b/src/modules/llms/api/openai.ts index 8663df91..c6af3454 100644 --- a/src/modules/llms/api/openai.ts +++ b/src/modules/llms/api/openai.ts @@ -79,7 +79,9 @@ export async function alterGeneratedImg ( } } -const prepareConversation = (conversation: ChatConversation[], model: string): ChatConversation[] => { +type ConversationOutput = Omit + +const prepareConversation = (conversation: ChatConversation[], model: string): ConversationOutput[] => { const messages = conversation.filter(c => c.model === model).map(m => { return { content: m.content, role: m.role } }) if (messages.length !== 1 || model === LlmModelsEnum.O1) { return messages @@ -125,7 +127,8 @@ export async function chatCompletion ( return { completion: { content: response.choices[0].message?.content ?? 'Error - no completion available', - role: 'assistant' + role: 'assistant', + timestamp: Date.now() }, usage: response.usage?.total_tokens, // 2010 price: price * config.openAi.chatGpt.priceAdjustment, @@ -215,7 +218,8 @@ export const streamChatCompletion = async ( return { completion: { content: completion, - role: 'assistant' + role: 'assistant', + timestamp: Date.now() }, usage: outputTokens + inputTokens, price: 0, @@ -308,7 +312,8 @@ export const streamChatVisionCompletion = async ( return { completion: { content: completion, - role: 'assistant' + role: 'assistant', + timestamp: Date.now() }, usage: outputTokens + inputTokens, price: 0, @@ -319,7 +324,7 @@ export const streamChatVisionCompletion = async ( export async function improvePrompt (promptText: string, model: string): Promise { const prompt = `Improve this picture description using max 100 words and don't add additional text to the image: ${promptText} ` - const conversation = [{ role: 'user', content: prompt }] + const conversation = [{ role: 'user', content: prompt, timestamp: Date.now() }] const response = await chatCompletion(conversation, model) return response.completion?.content as string ?? '' } diff --git a/src/modules/llms/api/pdfHandler.ts b/src/modules/llms/api/pdfHandler.ts index a11166c1..5b20bb6a 100644 --- a/src/modules/llms/api/pdfHandler.ts +++ b/src/modules/llms/api/pdfHandler.ts @@ -19,7 +19,8 @@ export const handlePdf = async (prompt: string): Promise => { return { completion: { content: response.data.response, - role: 'system' + role: 'system', + timestamp: Date.now() }, prompt, price: response.data.cost diff --git a/src/modules/llms/api/vertex.ts b/src/modules/llms/api/vertex.ts index f1ca2aa3..0c7a0b15 100644 --- a/src/modules/llms/api/vertex.ts +++ b/src/modules/llms/api/vertex.ts @@ -1,6 +1,6 @@ import axios, { type AxiosResponse } from 'axios' import config from '../../../config' -import { type OnMessageContext, type ChatConversation, type OnCallBackQueryData } from '../../types' +import { type OnMessageContext, type ChatConversation, type OnCallBackQueryData, type ChatConversationWithoutTimestamp } from '../../types' import { type LlmCompletion } from './llmApi' import { type Readable } from 'stream' import { GrammyError } from 'grammy' @@ -29,7 +29,7 @@ export const vertexCompletion = async ( stream: false, messages: conversation.filter(c => c.model === model) .map((msg) => { - const msgFiltered: ChatConversation = { content: msg.content, model: msg.model } + const msgFiltered: ChatConversationWithoutTimestamp = { content: msg.content, model: msg.model } if (model === LlmModelsEnum.CHAT_BISON) { msgFiltered.author = msg.role } else { @@ -48,7 +48,8 @@ export const vertexCompletion = async ( completion: { content: response.data._prediction_response[0][0].candidates[0].content, role: 'bot', // role replace to author attribute will be done later - model + model, + timestamp: Date.now() }, usage: totalOutputTokens + totalInputTokens, price: 0 @@ -145,7 +146,8 @@ export const vertexStreamCompletion = async ( completion: { content: completion, role: 'assistant', - model + model, + timestamp: Date.now() }, usage: parseInt(totalOutputTokens, 10) + parseInt(totalInputTokens, 10), price: 0, diff --git a/src/modules/llms/llmsBase.ts b/src/modules/llms/llmsBase.ts index bf2f8e43..bf33a4dc 100644 --- a/src/modules/llms/llmsBase.ts +++ b/src/modules/llms/llmsBase.ts @@ -37,6 +37,7 @@ import { type LLMModelsManager, type ModelVersion } from './utils/llmModelsManager' +import { conversationManager } from './utils/conversationManager' export abstract class LlmsBase implements PayableBot { public module: string @@ -205,7 +206,8 @@ export abstract class LlmsBase implements PayableBot { id: ctx.message?.message_id, model, content: await preparePrompt(ctx, prompt as string), - numSubAgents: 0 + numSubAgents: 0, + timestamp: Date.now() }) if (!session.isProcessingQueue) { session.isProcessingQueue = true @@ -218,7 +220,8 @@ export abstract class LlmsBase implements PayableBot { id: ctx.message?.message_id ?? ctx.message?.message_thread_id ?? 0, model, content: prompt as string ?? '', // await preparePrompt(ctx, prompt as string), - numSubAgents: supportedAgents + numSubAgents: supportedAgents, + timestamp: Date.now() } await this.runSubagents(ctx, msg, stream, usesTools) // prompt as string) } @@ -230,6 +233,8 @@ export abstract class LlmsBase implements PayableBot { async onChatRequestHandler (ctx: OnMessageContext | OnCallBackQueryData, stream: boolean, usesTools: boolean): Promise { const session = this.getSession(ctx) + session.chatConversation = conversationManager.manageConversationWindow(session.chatConversation) + while (session.requestQueue.length > 0) { try { const msg = session.requestQueue.shift() @@ -272,7 +277,8 @@ export abstract class LlmsBase implements PayableBot { const chat: ChatConversation = { content: enhancedPrompt || prompt, role: 'user', - model: modelVersion + model: modelVersion, + timestamp: Date.now() } chatConversation.push(chat) const payload = { @@ -358,7 +364,8 @@ export abstract class LlmsBase implements PayableBot { conversation.push({ role: 'assistant', content: completion.completion?.content ?? '', - model + model, + timestamp: Date.now() }) return { price: price.price, @@ -371,7 +378,8 @@ export abstract class LlmsBase implements PayableBot { conversation.push({ role: 'assistant', content: response.completion?.content ?? '', - model + model, + timestamp: Date.now() }) return { price: response.price, diff --git a/src/modules/llms/utils/conversationManager.ts b/src/modules/llms/utils/conversationManager.ts new file mode 100644 index 00000000..66be2a52 --- /dev/null +++ b/src/modules/llms/utils/conversationManager.ts @@ -0,0 +1,41 @@ +import { type VisionContent, type ChatConversation } from '../../types' + +const MINUTE_IN_MS = 60000 // 1 minute in milliseconds +const INACTIVE_THRESHOLD = 5 * MINUTE_IN_MS // 5 minutes +const IDLE_THRESHOLD = MINUTE_IN_MS // 1 minute +const IDLE_MESSAGE_LIMIT = 5 + +// const HOUR_IN_MS = 3600000 // 1 hour in milliseconds +// const INACTIVE_THRESHOLD = 12 * HOUR_IN_MS // 12 hours +// const IDLE_THRESHOLD = HOUR_IN_MS // 1 hour +// const IDLE_MESSAGE_LIMIT = 5 + +// Utility functions +export const conversationManager = { + manageConversationWindow (conversation: ChatConversation[]): ChatConversation[] { + console.log('fco::::::: here', conversation.length) + if (conversation.length === 0) return conversation + const now = Date.now() + const lastMessageTime = conversation[conversation.length - 1].timestamp + const timeDifference = now - lastMessageTime + // Case 1: Inactive conversation (>12 hours) - Reset + if (timeDifference > INACTIVE_THRESHOLD) { + return [] + } + + // Case 2: Idle conversation (>1 hour) - Keep last 5 messages + if (timeDifference > IDLE_THRESHOLD) { + return conversation.slice(-IDLE_MESSAGE_LIMIT) + } + + // Case 3: Active conversation (<1 hour) - Keep full history + return conversation + }, + + addMessageWithTimestamp (message: Omit | Partial> & { content: string | VisionContent[] }): ChatConversation { + return { + ...message, + timestamp: Date.now() + } + } +} diff --git a/src/modules/subagents/llamaSubagent.ts b/src/modules/subagents/llamaSubagent.ts index 28617980..3bbe58cd 100644 --- a/src/modules/subagents/llamaSubagent.ts +++ b/src/modules/subagents/llamaSubagent.ts @@ -11,7 +11,8 @@ import { type Collection, type OnCallBackQueryData, type OnMessageContext, type SubagentResult, - SubagentStatus + SubagentStatus, + type ChatConversationWithoutTimestamp } from '../types' import config from '../../config' import { appText } from '../../utils/text' @@ -305,7 +306,7 @@ export class LlamaAgent extends SubagentBase { const session = this.getSession(ctx) const collection = ctx.session.collections.activeCollections.find(c => c.url === url) if (collection) { - const conversation = this.getCollectionConversation(ctx, collection) + const conversation = this.getCollectionConversation(ctx, collection) as unknown as ChatConversationWithoutTimestamp[] if (conversation.length === 0) { conversation.push({ role: 'system', diff --git a/src/modules/types.ts b/src/modules/types.ts index 877a80c0..53588d4d 100644 --- a/src/modules/types.ts +++ b/src/modules/types.ts @@ -58,8 +58,11 @@ export interface ChatConversation { content: string | VisionContent[] model?: string numSubAgents?: number + timestamp: number } +export type ChatConversationWithoutTimestamp = Omit + export interface ImageRequest { command?: 'dalle' | 'alter' | 'vision' prompt?: string diff --git a/src/modules/voice-to-voice-gpt/index.ts b/src/modules/voice-to-voice-gpt/index.ts index 526639e8..46788a81 100644 --- a/src/modules/voice-to-voice-gpt/index.ts +++ b/src/modules/voice-to-voice-gpt/index.ts @@ -63,7 +63,7 @@ export class VoiceToVoiceGPTBot implements PayableBot { const resultText = await speechToText(fs.createReadStream(filename)) fs.rmSync(filename) - const conversation = [{ role: 'user', content: resultText }] + const conversation = [{ role: 'user', content: resultText, timestamp: Date.now() }] const response = await chatCompletion(conversation, LlmModelsEnum.GPT_35_TURBO) const voiceResult = await generateVoiceFromText(response.completion?.content as string) From 36a20e7250292dcec64341e169d364840d7f4561 Mon Sep 17 00:00:00 2001 From: fegloff Date: Mon, 4 Nov 2024 12:06:23 -0500 Subject: [PATCH 2/2] update sliding window logic to 3AM daily message reset --- src/bot.ts | 13 ++ src/helpers.ts | 7 +- src/modules/llms/llmsBase.ts | 46 ++++++- src/modules/llms/utils/conversationManager.ts | 119 ++++++++++++++---- src/modules/types.ts | 6 + 5 files changed, 160 insertions(+), 31 deletions(-) diff --git a/src/bot.ts b/src/bot.ts index 61b90e19..e4541553 100644 --- a/src/bot.ts +++ b/src/bot.ts @@ -498,6 +498,19 @@ const logErrorHandler = (ex: any): void => { logger.error(ex) } +// bot.command('testcleanup', async (ctx) => { +// await openAiBot.testCleanup(ctx as OnMessageContext) +// }) + +bot.command('new', async (ctx) => { + writeCommandLog(ctx as OnMessageContext).catch(logErrorHandler) + await openAiBot.onStop(ctx as OnMessageContext) + return await ctx.reply('Chat history reseted', { + parse_mode: 'Markdown', + message_thread_id: ctx.message?.message_thread_id + }) +}) + bot.command('more', async (ctx) => { writeCommandLog(ctx as OnMessageContext).catch(logErrorHandler) return await ctx.reply(commandsHelpText.more, { diff --git a/src/helpers.ts b/src/helpers.ts index 246c124b..98cf8431 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -1,4 +1,5 @@ import config from './config' +import { conversationManager } from './modules/llms/utils/conversationManager' import { LlmModelsEnum } from './modules/llms/utils/llmModelsManager' import { type DalleImageSize } from './modules/llms/utils/types' import { type BotSessionData } from './modules/types' @@ -26,7 +27,8 @@ export function createInitialSessionData (): BotSessionData { price: 0, usage: 0, isProcessingQueue: false, - requestQueue: [] + requestQueue: [], + cleanupState: conversationManager.initializeCleanupTimes() }, chatGpt: { model: config.llms.model, @@ -36,7 +38,8 @@ export function createInitialSessionData (): BotSessionData { price: 0, usage: 0, isProcessingQueue: false, - requestQueue: [] + requestQueue: [], + cleanupState: conversationManager.initializeCleanupTimes() }, dalle: { numImages: config.openAi.dalle.sessionDefault.numImages, diff --git a/src/modules/llms/llmsBase.ts b/src/modules/llms/llmsBase.ts index bf33a4dc..6be611b1 100644 --- a/src/modules/llms/llmsBase.ts +++ b/src/modules/llms/llmsBase.ts @@ -233,8 +233,7 @@ export abstract class LlmsBase implements PayableBot { async onChatRequestHandler (ctx: OnMessageContext | OnCallBackQueryData, stream: boolean, usesTools: boolean): Promise { const session = this.getSession(ctx) - session.chatConversation = conversationManager.manageConversationWindow(session.chatConversation) - + session.chatConversation = conversationManager.manageConversationWindow(session.chatConversation, ctx, this.sessionDataKey) while (session.requestQueue.length > 0) { try { const msg = session.requestQueue.shift() @@ -478,6 +477,49 @@ export abstract class LlmsBase implements PayableBot { session.price = 0 } + async testCleanup (ctx: OnMessageContext | OnCallBackQueryData): Promise { + const session = this.getSession(ctx) + // Force cleanup times for testing + const now = new Date() + const forcedCleanupTime = new Date(now) + forcedCleanupTime.setHours(2, 59, 0, 0) // Set to 2:59 AM + session.cleanupState = { + nextCleanupTime: forcedCleanupTime.getTime() + (60 * 1000), // 3 AM + lastCleanupTime: forcedCleanupTime.getTime() - (24 * 60 * 60 * 1000) // Yesterday 2:59 AM + } + console.log('Testing cleanup with forced times:', { + nextCleanup: new Date(session.cleanupState.nextCleanupTime).toLocaleString(), + lastCleanup: new Date(session.cleanupState.lastCleanupTime).toLocaleString(), + currentTime: now.toLocaleString() + }) + // Add some test messages with various timestamps + if (session.chatConversation.length === 0) { + const yesterday = new Date(now) + yesterday.setDate(yesterday.getDate() - 1) + session.chatConversation = [ + { + role: 'user', + content: 'Message from 2 days ago', + model: 'test', + timestamp: yesterday.getTime() - (24 * 60 * 60 * 1000) + }, + { + role: 'assistant', + content: 'Message from yesterday', + model: 'test', + timestamp: yesterday.getTime() + }, + { + role: 'user', + content: 'Message from today', + model: 'test', + timestamp: now.getTime() + } + ] + } + await this.onChatRequestHandler(ctx, false, false) + } + async onError ( ctx: OnMessageContext | OnCallBackQueryData, e: any, diff --git a/src/modules/llms/utils/conversationManager.ts b/src/modules/llms/utils/conversationManager.ts index 66be2a52..d337b2a2 100644 --- a/src/modules/llms/utils/conversationManager.ts +++ b/src/modules/llms/utils/conversationManager.ts @@ -1,41 +1,106 @@ -import { type VisionContent, type ChatConversation } from '../../types' - -const MINUTE_IN_MS = 60000 // 1 minute in milliseconds -const INACTIVE_THRESHOLD = 5 * MINUTE_IN_MS // 5 minutes -const IDLE_THRESHOLD = MINUTE_IN_MS // 1 minute -const IDLE_MESSAGE_LIMIT = 5 - -// const HOUR_IN_MS = 3600000 // 1 hour in milliseconds -// const INACTIVE_THRESHOLD = 12 * HOUR_IN_MS // 12 hours -// const IDLE_THRESHOLD = HOUR_IN_MS // 1 hour -// const IDLE_MESSAGE_LIMIT = 5 - -// Utility functions -export const conversationManager = { - manageConversationWindow (conversation: ChatConversation[]): ChatConversation[] { - console.log('fco::::::: here', conversation.length) - if (conversation.length === 0) return conversation +import { + type VisionContent, + type ChatConversation, + type OnMessageContext, + type OnCallBackQueryData, + type ConversationManagerState, + type BotSessionData, + type LlmsSessionData, + type ImageGenSessionData +} from '../../types' + +// Constants for time calculations +const MS_PER_DAY = 24 * 60 * 60 * 1000 +const CLEANUP_HOUR = 3 // 3 AM cleanup time + +const getSession = (ctx: OnMessageContext | OnCallBackQueryData, sessionDataKey: string): +LlmsSessionData & ImageGenSessionData => { + return ctx.session[sessionDataKey as keyof BotSessionData] as LlmsSessionData & ImageGenSessionData +} + +const conversationManager = { + /** + * Initialize or update cleanup timestamps + */ + initializeCleanupTimes (): ConversationManagerState { + const now = new Date() + const today3AM = new Date(now) + today3AM.setHours(CLEANUP_HOUR, 0, 0, 0) + + if (now.getTime() >= today3AM.getTime()) { + // If current time is past 3 AM, set next cleanup to tomorrow 3 AM + return { + nextCleanupTime: today3AM.getTime() + MS_PER_DAY, + lastCleanupTime: today3AM.getTime() + } + } else { + // If current time is before 3 AM, set next cleanup to today 3 AM + return { + nextCleanupTime: today3AM.getTime(), + lastCleanupTime: today3AM.getTime() - MS_PER_DAY + } + } + }, + + /** + * Check if cleanup is needed based on context + */ + needsCleanup (ctx: OnMessageContext | OnCallBackQueryData, sessionDataKey: string): boolean { const now = Date.now() - const lastMessageTime = conversation[conversation.length - 1].timestamp - const timeDifference = now - lastMessageTime - // Case 1: Inactive conversation (>12 hours) - Reset - if (timeDifference > INACTIVE_THRESHOLD) { - return [] + const session = getSession(ctx, sessionDataKey) + + // Initialize times if not set + if (!session.cleanupState || session.cleanupState.nextCleanupTime === 0) { + session.cleanupState = this.initializeCleanupTimes() } - // Case 2: Idle conversation (>1 hour) - Keep last 5 messages - if (timeDifference > IDLE_THRESHOLD) { - return conversation.slice(-IDLE_MESSAGE_LIMIT) + // Check if we've passed the next cleanup time + if (now >= session.cleanupState.nextCleanupTime) { + // Update cleanup times in session + session.cleanupState = { + lastCleanupTime: session.cleanupState.nextCleanupTime, + nextCleanupTime: session.cleanupState.nextCleanupTime + MS_PER_DAY + } + return true + } + + return false + }, + + /** + * Manage conversation window with context-aware cleanup + */ + manageConversationWindow (conversation: ChatConversation[], ctx: OnMessageContext | OnCallBackQueryData, sessionDataKey: string): ChatConversation[] { + if (conversation.length === 0) return conversation + + // Only perform cleanup if needed + if (this.needsCleanup(ctx, sessionDataKey)) { + const session = getSession(ctx, sessionDataKey) + return conversation.filter(msg => msg.timestamp >= session.cleanupState.lastCleanupTime) } - // Case 3: Active conversation (<1 hour) - Keep full history return conversation }, - addMessageWithTimestamp (message: Omit | Partial> & { content: string | VisionContent[] }): ChatConversation { + /** + * Add a new message to the conversation with current timestamp + */ + addMessageWithTimestamp ( + message: Omit | + Partial> & + { content: string | VisionContent[] }, + ctx: OnMessageContext | OnCallBackQueryData + ): ChatConversation { + // Initialize times if not set + if (!ctx.session.llms.cleanupState || ctx.session.llms.cleanupState.nextCleanupTime === 0) { + ctx.session.llms.cleanupState = this.initializeCleanupTimes() + } + return { ...message, timestamp: Date.now() } } } + +export { conversationManager } diff --git a/src/modules/types.ts b/src/modules/types.ts index 53588d4d..c60a7509 100644 --- a/src/modules/types.ts +++ b/src/modules/types.ts @@ -85,6 +85,11 @@ export interface promptRequest { commandPrefix?: string } +export interface ConversationManagerState { + lastCleanupTime: number + nextCleanupTime: number +} + export interface LlmsSessionData { model: string isEnabled: boolean @@ -94,6 +99,7 @@ export interface LlmsSessionData { price: number requestQueue: ChatConversation[] isProcessingQueue: boolean + cleanupState: ConversationManagerState } export interface OneCountryData {