feat: support add image to chat

packages/ai-native/src/browser/ai-core.contribution.ts

@@ -95,6 +95,7 @@ import { ChatProxyService } from './chat/chat-proxy.service';
 import { ChatInternalService } from './chat/chat.internal.service';
 import { AIChatView } from './chat/chat.view';
 import { CodeActionSingleHandler } from './contrib/code-action/code-action.handler';
+import { ImageUploadProviderRegistryToken } from './contrib/image-upload/image-upload.feature.registry';
 import { AIInlineCompletionsProvider } from './contrib/inline-completions/completeProvider';
 import { InlineCompletionsController } from './contrib/inline-completions/inline-completions.controller';
 import { AICompletionsService } from './contrib/inline-completions/service/ai-completions.service';
@@ -114,6 +115,7 @@ import {
   AINativeCoreContribution,
   IChatFeatureRegistry,
   IChatRenderRegistry,
+  IImageUploadProviderRegistry,
   IIntelligentCompletionsRegistry,
   IMCPServerRegistry,
   IProblemFixProviderRegistry,
@@ -196,6 +198,9 @@ export class AINativeBrowserContribution
   @Autowired(IntelligentCompletionsRegistryToken)
   private readonly intelligentCompletionsRegistry: IIntelligentCompletionsRegistry;
 
+  @Autowired(ImageUploadProviderRegistryToken)
+  private readonly imageUploadProviderRegistry: IImageUploadProviderRegistry;
+
   @Autowired(ProblemFixRegistryToken)
   private readonly problemFixProviderRegistry: IProblemFixProviderRegistry;
 
@@ -453,6 +458,7 @@ export class AINativeBrowserContribution
       contribution.registerIntelligentCompletionFeature?.(this.intelligentCompletionsRegistry);
       contribution.registerProblemFixFeature?.(this.problemFixProviderRegistry);
       contribution.registerChatAgentPromptProvider?.();
+      contribution.registerImageUploadProvider?.(this.imageUploadProviderRegistry);
     });
 
     // 注册 Opensumi 框架提供的 MCP Server Tools 能力 (此时的 Opensumi 作为 MCP Server)

packages/ai-native/src/browser/chat/chat-manager.service.ts

@@ -150,7 +150,7 @@ export class ChatManagerService extends Disposable {
     this.saveSessions();
   }
 
-  createRequest(sessionId: string, message: string, agentId: string, command?: string) {
+  createRequest(sessionId: string, message: string, agentId: string, command?: string, images?: string[]) {
     const model = this.getSession(sessionId);
     if (!model) {
       throw new Error(`Unknown session: ${sessionId}`);
@@ -160,7 +160,7 @@ export class ChatManagerService extends Disposable {
       return;
     }
 
-    return model.addRequest({ prompt: message, agentId, command });
+    return model.addRequest({ prompt: message, agentId, command, images });
   }
 
   async sendRequest(sessionId: string, request: ChatRequestModel, regenerate: boolean) {
@@ -191,6 +191,7 @@ export class ChatManagerService extends Disposable {
         requestId: request.requestId,
         message: request.message.prompt,
         command: request.message.command,
+        images: request.message.images,
         regenerate,
       };
       const result = await this.chatAgentService.invokeAgent(

packages/ai-native/src/browser/chat/chat-proxy.service.ts

@@ -151,6 +151,7 @@ export class ChatProxyService extends Disposable {
               requestId: request.requestId,
               sessionId: request.sessionId,
               history,
+              images: request.images,
               ...this.getRequestOptions(),
             },
             token,

packages/ai-native/src/browser/chat/chat.internal.service.ts

@@ -62,8 +62,8 @@ export class ChatInternalService extends Disposable {
     this._onChangeRequestId.fire(id);
   }
 
-  createRequest(input: string, agentId: string, command?: string) {
-    return this.chatManagerService.createRequest(this.#sessionModel.sessionId, input, agentId, command);
+  createRequest(input: string, agentId: string, images?: string[], command?: string) {
+    return this.chatManagerService.createRequest(this.#sessionModel.sessionId, input, agentId, command, images);
   }
 
   sendRequest(request: ChatRequestModel, regenerate = false) {

packages/ai-native/src/browser/chat/chat.view.tsx

@@ -281,7 +281,7 @@ export const AIChatView = () => {
           if (loading) {
             return;
           }
-          await handleSend(message.message, message.agentId, message.command);
+          await handleSend(message.message, message.images, message.agentId, message.command);
         } else {
           if (message.agentId) {
             setAgentId(message.agentId);
@@ -462,10 +462,16 @@ export const AIChatView = () => {
   );
 
   const renderUserMessage = React.useCallback(
-    async (renderModel: { message: string; agentId?: string; relationId: string; command?: string }) => {
+    async (renderModel: {
+      message: string;
+      images?: string[];
+      agentId?: string;
+      relationId: string;
+      command?: string;
+    }) => {
       const ChatUserRoleRender = chatRenderRegistry.chatUserRoleRender;
 
-      const { message, agentId, relationId, command } = renderModel;
+      const { message, images, agentId, relationId, command } = renderModel;
 
       const visibleAgentId = agentId === ChatProxyService.AGENT_ID ? '' : agentId;
 
@@ -474,12 +480,13 @@ export const AIChatView = () => {
           id: uuid(6),
           relationId,
           text: ChatUserRoleRender ? (
-            <ChatUserRoleRender content={message} agentId={visibleAgentId} command={command} />
+            <ChatUserRoleRender content={message} images={images} agentId={visibleAgentId} command={command} />
           ) : (
             <CodeBlockWrapperInput
               labelService={labelService}
               relationId={relationId}
               text={message}
+              images={images}
               agentId={visibleAgentId}
               command={command}
               workspaceService={workspaceService}
@@ -598,10 +605,10 @@ export const AIChatView = () => {
 
   const handleAgentReply = React.useCallback(
     async (value: IChatMessageStructure) => {
-      const { message, agentId, command, reportExtra } = value;
+      const { message, images, agentId, command, reportExtra } = value;
       const { actionType, actionSource } = reportExtra || {};
 
-      const request = aiChatService.createRequest(message, agentId!, command);
+      const request = aiChatService.createRequest(message, agentId!, images, command);
       if (!request) {
         return;
       }
@@ -627,6 +634,7 @@ export const AIChatView = () => {
 
       msgHistoryManager.addUserMessage({
         content: message,
+        images: images || [],
         agentId: agentId!,
         agentCommand: command!,
         relationId,
@@ -635,6 +643,7 @@ export const AIChatView = () => {
       await renderUserMessage({
         relationId,
         message,
+        images,
         command,
         agentId,
       });
@@ -668,7 +677,7 @@ export const AIChatView = () => {
   );
 
   const handleSend = React.useCallback(
-    async (message: string, agentId?: string, command?: string) => {
+    async (message: string, images?: string[], agentId?: string, command?: string) => {
       const reportExtra = {
         actionSource: ActionSourceEnum.Chat,
         actionType: ActionTypeEnum.Send,
@@ -707,7 +716,7 @@ export const AIChatView = () => {
           processedContent = processedContent.replace(match, `\`<attached_folder>${relativePath}\``);
         }
       }
-      return handleAgentReply({ message: processedContent, agentId, command, reportExtra });
+      return handleAgentReply({ message: processedContent, images, agentId, command, reportExtra });
     },
     [handleAgentReply],
   );
@@ -750,6 +759,7 @@ export const AIChatView = () => {
             message: msg.content,
             agentId: msg.agentId,
             command: msg.agentCommand,
+            images: msg.images,
           });
         } else if (msg.role === ChatMessageRole.Assistant && msg.requestId) {
           const request = aiChatService.sessionModel.getRequest(msg.requestId)!;

packages/ai-native/src/browser/components/ChatEditor.tsx

@@ -2,6 +2,7 @@ import capitalize from 'lodash/capitalize';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 import Highlight from 'react-highlight';
 
+import { Image } from '@opensumi/ide-components/lib/image';
 import {
   EDITOR_COMMANDS,
   FILE_COMMANDS,
@@ -336,6 +337,7 @@ export const CodeBlockWrapper = ({
 
 export const CodeBlockWrapperInput = ({
   text,
+  images,
   relationId,
   agentId,
   command,
@@ -344,6 +346,7 @@ export const CodeBlockWrapperInput = ({
   commandService,
 }: {
   text: string;
+  images?: string[];
   relationId: string;
   agentId?: string;
   command?: string;
@@ -370,6 +373,11 @@ export const CodeBlockWrapperInput = ({
 
   return (
     <div className={styles.ai_chat_code_wrapper}>
+      {images?.map((image) => (
+        <div className={styles.image_wrapper}>
+          <Image src={image} />
+        </div>
+      ))}
       <div className={styles.render_text}>
         {tag && (
           <div className={styles.tag_wrapper}>

packages/ai-native/src/browser/components/ChatInput.tsx

@@ -162,7 +162,7 @@ const AgentWidget = ({ agentId, command }) => (
 );
 
 export interface IChatInputProps {
-  onSend: (value: string, agentId?: string, command?: string) => void;
+  onSend: (value: string, images?: string[], agentId?: string, command?: string) => void;
   onValueChange?: (value: string) => void;
   onExpand?: (value: boolean) => void;
   placeholder?: string;
@@ -341,7 +341,7 @@ export const ChatInput = React.forwardRef((props: IChatInputProps, ref) => {
     }
 
     const handleSendLogic = (newValue: string = value) => {
-      onSend(newValue, agentId, command);
+      onSend(newValue, [], agentId, command);
       setValue('');
       setTheme('');
       setAgentId('');

packages/ai-native/src/browser/components/ChatMentionInput.tsx

@@ -1,23 +1,34 @@
+import { DataContent } from 'ai';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 
+import { Image } from '@opensumi/ide-components/lib/image';
 import { LabelService, RecentFilesManager, useInjectable } from '@opensumi/ide-core-browser';
-import { getIcon } from '@opensumi/ide-core-browser/lib/components';
+import { Icon, getIcon } from '@opensumi/ide-core-browser/lib/components';
 import { URI, localize } from '@opensumi/ide-core-common';
 import { CommandService } from '@opensumi/ide-core-common/lib/command';
 import { WorkbenchEditorService } from '@opensumi/ide-editor';
 import { FileSearchServicePath, IFileSearchService } from '@opensumi/ide-file-search';
+import { IMessageService } from '@opensumi/ide-overlay';
 import { IWorkspaceService } from '@opensumi/ide-workspace';
 
 import { IChatInternalService } from '../../common';
 import { ChatInternalService } from '../chat/chat.internal.service';
+import { ImageUploadProviderRegistryToken } from '../contrib/image-upload/image-upload.feature.registry';
 import { OPEN_MCP_CONFIG_COMMAND } from '../mcp/config/mcp-config.commands';
+import { IImageUploadProviderRegistry } from '../types';
 
 import styles from './components.module.less';
 import { MentionInput } from './mention-input/mention-input';
 import { FooterButtonPosition, FooterConfig, MentionItem, MentionType } from './mention-input/types';
 
 export interface IChatMentionInputProps {
-  onSend: (value: string, agentId?: string, command?: string, option?: { model: string; [key: string]: any }) => void;
+  onSend: (
+    value: string,
+    images?: string[],
+    agentId?: string,
+    command?: string,
+    option?: { model: string; [key: string]: any },
+  ) => void;
   onValueChange?: (value: string) => void;
   onExpand?: (value: boolean) => void;
   placeholder?: string;
@@ -26,6 +37,7 @@ export interface IChatMentionInputProps {
   sendBtnClassName?: string;
   defaultHeight?: number;
   value?: string;
+  images?: Array<DataContent | URL>;
   autoFocus?: boolean;
   theme?: string | null;
   setTheme: (theme: string | null) => void;
@@ -41,13 +53,16 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
   const { onSend, disabled = false } = props;
 
   const [value, setValue] = useState(props.value || '');
+  const [images, setImages] = useState(props.images || []);
   const aiChatService = useInjectable<ChatInternalService>(IChatInternalService);
   const commandService = useInjectable<CommandService>(CommandService);
   const searchService = useInjectable<IFileSearchService>(FileSearchServicePath);
   const recentFilesManager = useInjectable<RecentFilesManager>(RecentFilesManager);
   const workspaceService = useInjectable<IWorkspaceService>(IWorkspaceService);
   const editorService = useInjectable<WorkbenchEditorService>(WorkbenchEditorService);
   const labelService = useInjectable<LabelService>(LabelService);
+  const messageService = useInjectable<IMessageService>(IMessageService);
+  const imageUploadProviderRegistry = useInjectable<IImageUploadProviderRegistry>(ImageUploadProviderRegistryToken);
 
   const handleShowMCPConfig = React.useCallback(() => {
     commandService.executeCommand(OPEN_MCP_CONFIG_COMMAND.id);
@@ -239,6 +254,24 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
           onClick: handleShowMCPConfig,
           position: FooterButtonPosition.LEFT,
         },
+        {
+          id: 'upload-image',
+          iconClass: 'codicon codicon-file-media',
+          title: 'Upload Image',
+          onClick: () => {
+            const input = document.createElement('input');
+            input.type = 'file';
+            input.accept = 'image/*';
+            input.onchange = (e) => {
+              const file = (e.target as HTMLInputElement).files?.[0];
+              if (file) {
+                handleImageUpload(file);
+              }
+            };
+            input.click();
+          },
+          position: FooterButtonPosition.LEFT,
+        },
       ],
       showModelSelector: true,
     }),
@@ -254,13 +287,46 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
       if (disabled) {
         return;
       }
-      onSend(content, undefined, undefined, option);
+      onSend(
+        content,
+        images.map((image) => image.toString()),
+        undefined,
+        undefined,
+        option,
+      );
     },
-    [onSend, editorService, disabled],
+    [onSend, images, disabled],
+  );
+
+  const handleImageUpload = useCallback(
+    async (file: File) => {
+      const allowedTypes = ['image/jpeg', 'image/jpg', 'image/png', 'image/webp', 'image/gif'];
+      if (!allowedTypes.includes(file.type)) {
+        messageService.error('Only JPG, PNG, WebP and GIF images are supported');
+        return;
+      }
+
+      const imageUploadProvider = imageUploadProviderRegistry.getImageUploadProvider();
+      if (!imageUploadProvider) {
+        messageService.error('No image upload provider found');
+        return;
+      }
+      const data = await imageUploadProvider.imageUpload(file);
+      setImages([...images, data]);
+    },
+    [images],
+  );
+
+  const handleDeleteImage = useCallback(
+    (index: number) => {
+      setImages(images.filter((_, i) => i !== index));
+    },
+    [images],
   );
 
   return (
     <div className={styles.chat_input_container}>
+      {images.length > 0 && <ImagePreviewer images={images} onDelete={handleDeleteImage} />}
       <MentionInput
         mentionItems={defaultMenuItems}
         onSend={handleSend}
@@ -270,7 +336,29 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
         workspaceService={workspaceService}
         placeholder={localize('aiNative.chat.input.placeholder.default')}
         footerConfig={defaultMentionInputFooterOptions}
+        onImageUpload={handleImageUpload}
       />
     </div>
   );
 };
+
+const ImagePreviewer = ({
+  images,
+  onDelete,
+}: {
+  images: Array<DataContent | URL>;
+  onDelete: (index: number) => void;
+}) => (
+  <div>
+    <div className={styles.thumbnail_container}>
+      {images.map((image, index) => (
+        <div key={index} className={styles.thumbnail}>
+          <Image src={image.toString()} />
+          <button onClick={() => onDelete(index)} className={styles.delete_button}>
+            <Icon iconClass='codicon codicon-close' />
+          </button>
+        </div>
+      ))}
+    </div>
+  </div>
+);