Skip to content

Commit

Permalink
feat(FR-255): synchronize the file input in the LLM Playground. (#3032)
Browse files Browse the repository at this point in the history
resolves #2861 ([FR-255](https://lablup.atlassian.net/browse/FR-255?atlOrigin=eyJpIjoiODA5YzI1OGQ2OWJhNGM5NThmMWFmNWU3NWVhY2QwZjYiLCJwIjoiaiJ9))

Adds file attachment synchronization support to the LLM chat interface, enabling consistent file handling across multiple chat instances. When files are attached in one chat window, they are now properly synchronized across all active chat sessions.

**Changes:**
- Introduces `synchronizedAttachmentState` atom for managing shared file attachments
- Adds file attachment handling in `EndpointLLMChatCard`
- Implements `createDataTransferFiles` utility function for consistent file processing
- Updates attachment state management in `LLMChatCard`

**Checklist:**
- [ ] Documentation
- [ ] Minium required manager version
- [x] Specific setting for review:
  - Test file attachments in synchronized chat mode
  - Verify attachments persist across chat instances
- [x] Minimum requirements to check during review:
  - File attachments sync correctly between chat windows
  - Attachments clear properly after submission
  - Supported file types (image/*, text/*) work as expected
- [x] Test cases:
  1. Attach files in one chat window, verify sync in others
  2. Submit message with attachments, confirm proper clearing
  3. Test with various file types to ensure compatibility

**Screenshots:**
![image.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/2HueYSdFvL8pOB5mgrUQ/15366be8-81ac-4efb-8fe3-60b61985bd19.png)]

[FR-255]: https://lablup.atlassian.net/browse/FR-255?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ
  • Loading branch information
agatha197 committed Feb 18, 2025
1 parent 322f54f commit 9b5b7b2
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 23 deletions.
10 changes: 10 additions & 0 deletions react/src/components/lablupTalkativotUI/EndpointLLMChatCard.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { Model } from './ChatUIModal';
import LLMChatCard, { BAIModel } from './LLMChatCard';
import { EndpointLLMChatCard_endpoint$key } from './__generated__/EndpointLLMChatCard_endpoint.graphql';
import { CloseOutlined, ReloadOutlined } from '@ant-design/icons';
import { AttachmentsProps } from '@ant-design/x';
import { Alert, Button, CardProps, Popconfirm, theme } from 'antd';
import graphql from 'babel-plugin-relay/macro';
import { atom, useAtom } from 'jotai';
Expand All @@ -14,6 +15,7 @@ import { useTranslation } from 'react-i18next';
import { useFragment } from 'react-relay';

const synchronizedMessageState = atom<string>('');
const synchronizedAttachmentState = atom<AttachmentsProps['items']>();
const chatSubmitKeyInfoState = atom<{ id: string; key: string } | undefined>(
undefined,
);
Expand Down Expand Up @@ -58,6 +60,9 @@ const EndpointLLMChatCard: React.FC<EndpointLLMChatCardProps> = ({
const [synchronizedMessage, setSynchronizedMessage] = useAtom(
synchronizedMessageState,
);
const [synchronizedAttachment, setSynchronizedAttachment] = useAtom(
synchronizedAttachmentState,
);

const [chatSubmitKeyInfo, setChatSubmitKeyInfo] = useAtom(
chatSubmitKeyInfoState,
Expand Down Expand Up @@ -169,11 +174,16 @@ const EndpointLLMChatCard: React.FC<EndpointLLMChatCardProps> = ({
onInputChange={(v) => {
setSynchronizedMessage(v);
}}
inputAttachment={isSynchronous ? synchronizedAttachment : undefined}
onAttachmentChange={(attachment) => {
setSynchronizedAttachment(attachment);
}}
submitKey={
chatSubmitKeyInfo?.id === submitId ? undefined : chatSubmitKeyInfo?.key
}
onSubmitChange={() => {
setSynchronizedMessage('');
setSynchronizedAttachment([]);
if (isSynchronous) {
setChatSubmitKeyInfo({
id: submitId,
Expand Down
73 changes: 50 additions & 23 deletions react/src/components/lablupTalkativotUI/LLMChatCard.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use client';

import { filterEmptyItem } from '../../helper';
import { createDataTransferFiles, filterEmptyItem } from '../../helper';
import { useWebUINavigate } from '../../hooks';
import { useTokenCount } from '../../hooks/useTokenizer';
import Flex from '../Flex';
Expand All @@ -18,7 +18,12 @@ import {
} from '@ant-design/icons';
import { Attachments, AttachmentsProps, Sender } from '@ant-design/x';
import { useControllableValue } from 'ahooks';
import { streamText, extractReasoningMiddleware, wrapLanguageModel } from 'ai';
import {
streamText,
extractReasoningMiddleware,
wrapLanguageModel,
ChatRequestOptions,
} from 'ai';
import {
Alert,
Badge,
Expand Down Expand Up @@ -63,10 +68,12 @@ export interface LLMChatCardProps extends CardProps {
alert?: React.ReactNode;
leftExtra?: React.ReactNode;
inputMessage?: string;
inputAttachment?: AttachmentsProps['items'];
submitKey?: string;
onAgentChange?: (agentId: string) => void;
onModelChange?: (modelId: string) => void;
onInputChange?: (input: string) => void;
onAttachmentChange?: (attachment: AttachmentsProps['items']) => void;
onSubmitChange?: () => void;
showCompareMenuItem?: boolean;
modelToken?: string;
Expand All @@ -84,15 +91,18 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
alert,
leftExtra,
inputMessage,
inputAttachment,
submitKey,
onInputChange,
onAttachmentChange,
onSubmitChange,
showCompareMenuItem,
modelToken,
...cardProps
}) => {
const webuiNavigate = useWebUINavigate();
const [isOpenAttachments, setIsOpenAttachments] = useState(false);
const [files, setFiles] = useState<AttachmentsProps['items']>([]);

const [modelId, setModelId] = useControllableValue(cardProps, {
valuePropName: 'modelId',
Expand Down Expand Up @@ -165,12 +175,35 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
}
}, [inputMessage, setInput]);

const setFilesFromInputAttachment = (
inputAttachment: AttachmentsProps['items'],
) => {
if (!_.isUndefined(inputAttachment) && !_.isEqual(files, inputAttachment)) {
setFiles(inputAttachment);
setIsOpenAttachments(true);
}
};

// If the `inputAttachment` prop exists, the `files` state has to follow it.
useEffect(() => {
setFilesFromInputAttachment(inputAttachment);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [inputAttachment]);

useEffect(() => {
if (!_.isUndefined(submitKey) && input) {
append({
role: 'user',
content: input,
});
const chatRequestOptions: ChatRequestOptions = {};
if (!_.isEmpty(files)) {
chatRequestOptions.experimental_attachments =
createDataTransferFiles(files);
}
append(
{
role: 'user',
content: input,
},
chatRequestOptions,
);
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [submitKey]);
Expand Down Expand Up @@ -199,8 +232,6 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
: 0;
}, [lastAssistantTokenCount, startTime]);

const [files, setFiles] = useState<AttachmentsProps['items']>([]);

const items: MenuProps['items'] = filterEmptyItem([
showCompareMenuItem && {
key: 'compare',
Expand Down Expand Up @@ -315,7 +346,10 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
getDropContainer={() => cardRef.current}
accept="image/*,text/*"
items={files}
onChange={({ fileList }) => setFiles(fileList)}
onChange={({ fileList }) => {
setFiles(fileList);
onAttachmentChange?.(fileList);
}}
placeholder={(type) =>
type === 'drop'
? {
Expand All @@ -338,6 +372,7 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
items={files}
onChange={({ fileList }) => {
setFiles(fileList);
onAttachmentChange?.(fileList);
setIsOpenAttachments(true);
}}
placeholder={(type) =>
Expand Down Expand Up @@ -369,25 +404,17 @@ const LLMChatCard: React.FC<LLMChatCardProps> = ({
}}
onSend={() => {
if (input || !_.isEmpty(files)) {
const fileList = _.map(
files,
(item) => item.originFileObj as File,
);
// Filter after converting to `File`
const fileListArray = _.filter(fileList, Boolean);
const dataTransfer = new DataTransfer();
_.forEach(fileListArray, (file) => {
dataTransfer.items.add(file);
});

const chatRequestOptions: ChatRequestOptions = {};
if (!_.isEmpty(files)) {
chatRequestOptions.experimental_attachments =
createDataTransferFiles(files);
}
append(
{
role: 'user',
content: input,
},
{
experimental_attachments: dataTransfer.files,
},
chatRequestOptions,
);

setTimeout(() => {
Expand Down
10 changes: 10 additions & 0 deletions react/src/helper/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { CommittedImage } from '../components/CustomizedImageList';
import { Image } from '../components/ImageEnvironmentSelectFormItems';
import { EnvironmentImage } from '../components/ImageList';
import { useSuspendedBackendaiClient } from '../hooks';
import { AttachmentsProps } from '@ant-design/x';
import { SorterResult } from 'antd/es/table/interface';
import dayjs from 'dayjs';
import { Duration } from 'dayjs/plugin/duration';
Expand Down Expand Up @@ -502,3 +503,12 @@ export const handleRowSelectionChange = <T extends object, K extends keyof T>(
);
});
};

export function createDataTransferFiles(files: AttachmentsProps['items']) {
const fileList = _.map(files, (item) => item.originFileObj as File);
const dataTransfer = new DataTransfer();
_.forEach(fileList, (file) => {
dataTransfer.items.add(file);
});
return dataTransfer.files;
}

0 comments on commit 9b5b7b2

Please sign in to comment.