Merge pull request #49 from animalnots/dev

v.1.7.0

.gitignore

@@ -25,4 +25,5 @@ dist-ssr
 
 release/
 
-.env
+.env
+models.json

package.json

@@ -1,7 +1,7 @@
 {
   "name": "better-chatgpt",
   "private": true,
-  "version": "1.7.0",
+  "version": "1.8.0",
   "type": "module",
   "homepage": "./",
   "main": "electron/index.cjs",

public/locales/en/import.json

@@ -1,10 +1,15 @@
 {
     "notifications": {
-        "invalidOpenAIDataFormat" : "Invalid openai data format",
-        "invalidChatsDataFormat" : "Invalid chats data format",
-        "invalidFormatForVersion" : "Invalid format for specified version",
-        "successfulImport" : "Succesfully imported!",
-        "unrecognisedDataFormat" : "Unrecognised data format. Supported formats are: BetterGPT export, OpenAI export, OpenAI Playground (JSON)"
-    }
-}
+      "invalidOpenAIDataFormat": "Invalid openai data format",
+      "invalidChatsDataFormat": "Invalid chats data format",
+      "invalidFormatForVersion": "Invalid format for specified version",
+      "quotaExceeded": "Storage quota exceeded",
+      "textSavedOnly": "Only text was saved",
+      "successfulImport": "Successfully imported!",
+      "nothingImported": "No data has been imported",
+      "unrecognisedDataFormat": "Unrecognised data format. Supported formats are: BetterGPT export, OpenAI export, OpenAI Playground (JSON)",
+      "chatsImported": "{{imported}} chats were imported out of {{total}}."
+    },
+    "reduceMessagesSuccess": "{{count}} messages were reduced."
+  }
 

public/locales/en/main.json

@@ -47,5 +47,10 @@
   "cloneChat": "Clone Chat",
   "cloned": "Cloned",
   "enterToSubmit": "Enter to submit",
-  "submitPlaceholder": "Type a message or click [/] for prompts..."
+  "submitPlaceholder": "Type a message or click [/] for prompts...",
+  "reduceMessagesWarning": "Reducing messages may result in data loss. It is recommended to download the chat in JSON format if you care about the data. Do you want to proceed?",
+  "reduceMessagesFailedImportWarning": "Full import failed as the data hit the maximum storage limit. Import as much as possible?",
+  "reduceMessagesButton": "Reduce Messages",
+  "reduceMessagesSuccess": "Successfully reduced messages. {{count}} messages were removed.",
+  "hiddenMessagesWarning": "Some messages were hidden with the total length of {{hiddenTokens}} tokens to {{reduceMessagesToTotalToken}} tokens to avoid laggy UI."
 }

sortModelsJsonKeys.js

@@ -0,0 +1,43 @@
+import fs from 'fs/promises';
+
+// Function to recursively sort object keys
+function sortObjectKeys(obj) {
+    if (Array.isArray(obj)) {
+        return obj.map(sortObjectKeys);
+    } else if (obj !== null && typeof obj === 'object') {
+        return Object.keys(obj).sort().reduce((sortedObj, key) => {
+            sortedObj[key] = sortObjectKeys(obj[key]);
+            return sortedObj;
+        }, {});
+    }
+    return obj;
+}
+
+// Read the JSON file
+async function processJsonFile(inputFilePath, outputFilePath) {
+    try {
+        const data = await fs.readFile(inputFilePath, 'utf8');
+
+        // Parse the JSON data
+        const jsonData = JSON.parse(data);
+
+        // Sort the JSON data
+        const sortedJsonData = sortObjectKeys(jsonData);
+
+        // Convert the sorted JSON data back to a string
+        const sortedJsonString = JSON.stringify(sortedJsonData, null, 2);
+
+        // Write the sorted JSON data to a new file
+        await fs.writeFile(outputFilePath, sortedJsonString, 'utf8');
+        console.log('File has been saved with sorted keys.');
+    } catch (err) {
+        console.error('Error processing the file:', err);
+    }
+}
+
+// Define file paths
+const inputFilePath = 'models.json';
+const outputFilePath = 'public/models.json';
+
+// Process the JSON file
+processJsonFile(inputFilePath, outputFilePath);

src/components/Chat/ChatContent/ChatContent.tsx

@@ -1,6 +1,7 @@
 import React, { useEffect, useRef } from 'react';
 import ScrollToBottom from 'react-scroll-to-bottom';
 import useStore from '@store/store';
+import { useTranslation } from 'react-i18next';
 
 import ScrollToBottomButton from './ScrollToBottomButton';
 import ChatTitle from './ChatTitle';
@@ -13,10 +14,15 @@ import DownloadChat from './DownloadChat';
 import CloneChat from './CloneChat';
 import ShareGPT from '@components/ShareGPT';
 import { ImageContentInterface, TextContentInterface } from '@type/chat';
+import countTokens, { limitMessageTokens } from '@utils/messageUtils';
+import { defaultModel, reduceMessagesToTotalToken } from '@constants/chat';
+import { toast } from 'react-toastify';
 
 const ChatContent = () => {
+  const { t } = useTranslation();
   const inputRole = useStore((state) => state.inputRole);
   const setError = useStore((state) => state.setError);
+  const setChats = useStore((state) => state.setChats);
   const messages = useStore((state) =>
     state.chats &&
     state.chats.length > 0 &&
@@ -25,6 +31,7 @@ const ChatContent = () => {
       ? state.chats[state.currentChatIndex].messages
       : []
   );
+  const currentChatIndex = useStore((state) => state.currentChatIndex);
   const stickyIndex = useStore((state) =>
     state.chats &&
     state.chats.length > 0 &&
@@ -36,6 +43,50 @@ const ChatContent = () => {
   const advancedMode = useStore((state) => state.advancedMode);
   const generating = useStore.getState().generating;
   const hideSideMenu = useStore((state) => state.hideSideMenu);
+  const model = useStore((state) =>
+    state.chats &&
+    state.chats.length > 0 &&
+    state.currentChatIndex >= 0 &&
+    state.currentChatIndex < state.chats.length
+      ? state.chats[state.currentChatIndex].config.model
+      : defaultModel
+  );
+  const messagesLimited = limitMessageTokens(messages, reduceMessagesToTotalToken, model);
+
+  const handleReduceMessages = () => {
+    const confirmMessage = t('reduceMessagesWarning');
+    if (window.confirm(confirmMessage)) {
+      const updatedChats = JSON.parse(JSON.stringify(useStore.getState().chats));
+      const removedMessagesCount = messages.length - messagesLimited.length;
+      updatedChats[currentChatIndex].messages = messagesLimited;
+      setChats(updatedChats);
+      toast.dismiss();
+      toast.success(t('reduceMessagesSuccess', { count: removedMessagesCount }));
+    }
+  };
+
+  useEffect(() => {
+    if (!generating) {
+      if (messagesLimited.length < messages.length) {
+        const hiddenTokens =
+          countTokens(messages, model) - countTokens(messagesLimited, model);
+        const message = (
+          <div>
+            <span>
+              {t('hiddenMessagesWarning', { hiddenTokens, reduceMessagesToTotalToken })}
+            </span><br />
+            <button
+              onClick={handleReduceMessages}
+              className="px-2 py-1 bg-blue-500 text-white rounded"
+            >
+              {t('reduceMessagesButton')}
+            </button>
+          </div>
+        );
+        toast.error(message);
+      }
+    }
+  }, [messagesLimited, generating, messages, model]);
 
   const saveRef = useRef<HTMLDivElement>(null);
 
@@ -64,26 +115,29 @@ const ChatContent = () => {
             {!generating && advancedMode && messages?.length === 0 && (
               <NewMessageButton messageIndex={-1} />
             )}
-            {messages?.map((message, index) => (
-              (advancedMode || index !== 0 || message.role !== 'system') && (
-                <React.Fragment key={index}>
-                  <Message
-                    role={message.role}
-                    content={message.content}
-                    messageIndex={index}
-                  />
-                  {!generating && advancedMode && <NewMessageButton messageIndex={index} />}
-                </React.Fragment>
-              )
-            ))}
+            {messagesLimited?.map(
+              (message, index) =>
+                (advancedMode || index !== 0 || message.role !== 'system') && (
+                  <React.Fragment key={index}>
+                    <Message
+                      role={message.role}
+                      content={message.content}
+                      messageIndex={index}
+                    />
+                    {!generating && advancedMode && (
+                      <NewMessageButton messageIndex={index} />
+                    )}
+                  </React.Fragment>
+                )
+            )}
           </div>
 
           <Message
             role={inputRole}
             // For now we always initizlize a new message with an empty text content.
-            // It is possible to send a message to the API without a TextContentInterface, 
+            // It is possible to send a message to the API without a TextContentInterface,
             // but the UI would need to be modified to allow the user to control the order of text and image content
-            content={[{type: 'text', text: ''} as TextContentInterface]}
+            content={[{ type: 'text', text: '' } as TextContentInterface]}
             messageIndex={stickyIndex}
             sticky
           />