diff --git a/packages/ai-native/src/browser/chat/chat-manager.service.ts b/packages/ai-native/src/browser/chat/chat-manager.service.ts index 65258dcb5d..f7ee31cd24 100644 --- a/packages/ai-native/src/browser/chat/chat-manager.service.ts +++ b/packages/ai-native/src/browser/chat/chat-manager.service.ts @@ -150,7 +150,7 @@ export class ChatManagerService extends Disposable { this.saveSessions(); } - createRequest(sessionId: string, message: string, agentId: string, command?: string) { + createRequest(sessionId: string, message: string, agentId: string, command?: string, images?: string[]) { const model = this.getSession(sessionId); if (!model) { throw new Error(`Unknown session: ${sessionId}`); @@ -160,7 +160,7 @@ export class ChatManagerService extends Disposable { return; } - return model.addRequest({ prompt: message, agentId, command }); + return model.addRequest({ prompt: message, agentId, command, images }); } async sendRequest(sessionId: string, request: ChatRequestModel, regenerate: boolean) { @@ -191,6 +191,7 @@ export class ChatManagerService extends Disposable { requestId: request.requestId, message: request.message.prompt, command: request.message.command, + images: request.message.images, regenerate, }; const result = await this.chatAgentService.invokeAgent( diff --git a/packages/ai-native/src/browser/chat/chat-model.ts b/packages/ai-native/src/browser/chat/chat-model.ts index 2b85fd8ba8..6f87369df0 100644 --- a/packages/ai-native/src/browser/chat/chat-model.ts +++ b/packages/ai-native/src/browser/chat/chat-model.ts @@ -27,7 +27,7 @@ import { import { MsgHistoryManager } from '../model/msg-history-manager'; import { IChatSlashCommandItem } from '../types'; -import type { TextPart, ToolCallPart } from 'ai'; +import type { ImagePart, TextPart, ToolCallPart } from 'ai'; export type IChatProgressResponseContent = | IChatMarkdownContent @@ -330,7 +330,15 @@ export class ChatModel extends Disposable implements IChatModel { if (!request.response.isComplete) { continue; } - history.push({ role: 'user', content: request.message.prompt }); + history.push({ + role: 'user', + content: request.message.images?.length + ? [ + { type: 'text', text: request.message.prompt }, + ...request.message.images.map((image) => ({ type: 'image', image: new URL(image) } as ImagePart)), + ] + : request.message.prompt, + }); for (const part of request.response.responseParts) { if (part.kind === 'treeData' || part.kind === 'component') { continue; diff --git a/packages/ai-native/src/browser/chat/chat-proxy.service.ts b/packages/ai-native/src/browser/chat/chat-proxy.service.ts index bc50d6a9a0..307266c8bb 100644 --- a/packages/ai-native/src/browser/chat/chat-proxy.service.ts +++ b/packages/ai-native/src/browser/chat/chat-proxy.service.ts @@ -151,6 +151,7 @@ export class ChatProxyService extends Disposable { requestId: request.requestId, sessionId: request.sessionId, history, + images: request.images, ...this.getRequestOptions(), }, token, diff --git a/packages/ai-native/src/browser/chat/chat.feature.registry.ts b/packages/ai-native/src/browser/chat/chat.feature.registry.ts index 070a4b2896..e3b57b52b6 100644 --- a/packages/ai-native/src/browser/chat/chat.feature.registry.ts +++ b/packages/ai-native/src/browser/chat/chat.feature.registry.ts @@ -2,7 +2,7 @@ import { Injectable } from '@opensumi/di'; import { Disposable, Emitter, Event, getDebugLogger } from '@opensumi/ide-core-common'; import { IChatWelcomeMessageContent, ISampleQuestions, SLASH_SYMBOL } from '../../common'; -import { IChatFeatureRegistry, IChatSlashCommandHandler, IChatSlashCommandItem } from '../types'; +import { IChatFeatureRegistry, IChatSlashCommandHandler, IChatSlashCommandItem, IImageUploadProvider } from '../types'; import { ChatSlashCommandItemModel, ChatWelcomeMessageModel } from './chat-model'; import { ChatProxyService } from './chat-proxy.service'; @@ -12,6 +12,15 @@ export class ChatFeatureRegistry extends Disposable implements IChatFeatureRegis private readonly logger = getDebugLogger(); private slashCommandsMap: Map = new Map(); private slashCommandsHandlerMap: Map = new Map(); + private imageUploadProvider: IImageUploadProvider | undefined; + + public registerImageUploadProvider(provider: IImageUploadProvider): void { + this.imageUploadProvider = provider; + } + + public getImageUploadProvider(): IImageUploadProvider | undefined { + return this.imageUploadProvider; + } public chatWelcomeMessageModel?: ChatWelcomeMessageModel; diff --git a/packages/ai-native/src/browser/chat/chat.internal.service.ts b/packages/ai-native/src/browser/chat/chat.internal.service.ts index bb14c9d948..d8196dea6c 100644 --- a/packages/ai-native/src/browser/chat/chat.internal.service.ts +++ b/packages/ai-native/src/browser/chat/chat.internal.service.ts @@ -62,8 +62,8 @@ export class ChatInternalService extends Disposable { this._onChangeRequestId.fire(id); } - createRequest(input: string, agentId: string, command?: string) { - return this.chatManagerService.createRequest(this.#sessionModel.sessionId, input, agentId, command); + createRequest(input: string, agentId: string, images?: string[], command?: string) { + return this.chatManagerService.createRequest(this.#sessionModel.sessionId, input, agentId, command, images); } sendRequest(request: ChatRequestModel, regenerate = false) { diff --git a/packages/ai-native/src/browser/chat/chat.view.tsx b/packages/ai-native/src/browser/chat/chat.view.tsx index 462bb11a3e..1f701915bf 100644 --- a/packages/ai-native/src/browser/chat/chat.view.tsx +++ b/packages/ai-native/src/browser/chat/chat.view.tsx @@ -281,7 +281,7 @@ export const AIChatView = () => { if (loading) { return; } - await handleSend(message.message, message.agentId, message.command); + await handleSend(message.message, message.images, message.agentId, message.command); } else { if (message.agentId) { setAgentId(message.agentId); @@ -462,10 +462,16 @@ export const AIChatView = () => { ); const renderUserMessage = React.useCallback( - async (renderModel: { message: string; agentId?: string; relationId: string; command?: string }) => { + async (renderModel: { + message: string; + images?: string[]; + agentId?: string; + relationId: string; + command?: string; + }) => { const ChatUserRoleRender = chatRenderRegistry.chatUserRoleRender; - const { message, agentId, relationId, command } = renderModel; + const { message, images, agentId, relationId, command } = renderModel; const visibleAgentId = agentId === ChatProxyService.AGENT_ID ? '' : agentId; @@ -474,12 +480,13 @@ export const AIChatView = () => { id: uuid(6), relationId, text: ChatUserRoleRender ? ( - + ) : ( { const handleAgentReply = React.useCallback( async (value: IChatMessageStructure) => { - const { message, agentId, command, reportExtra } = value; + const { message, images, agentId, command, reportExtra } = value; const { actionType, actionSource } = reportExtra || {}; - const request = aiChatService.createRequest(message, agentId!, command); + const request = aiChatService.createRequest(message, agentId!, images, command); if (!request) { return; } @@ -627,6 +634,7 @@ export const AIChatView = () => { msgHistoryManager.addUserMessage({ content: message, + images: images || [], agentId: agentId!, agentCommand: command!, relationId, @@ -635,6 +643,7 @@ export const AIChatView = () => { await renderUserMessage({ relationId, message, + images, command, agentId, }); @@ -668,7 +677,7 @@ export const AIChatView = () => { ); const handleSend = React.useCallback( - async (message: string, agentId?: string, command?: string) => { + async (message: string, images?: string[], agentId?: string, command?: string) => { const reportExtra = { actionSource: ActionSourceEnum.Chat, actionType: ActionTypeEnum.Send, @@ -707,7 +716,7 @@ export const AIChatView = () => { processedContent = processedContent.replace(match, `\`${relativePath}\``); } } - return handleAgentReply({ message: processedContent, agentId, command, reportExtra }); + return handleAgentReply({ message: processedContent, images, agentId, command, reportExtra }); }, [handleAgentReply], ); @@ -750,6 +759,7 @@ export const AIChatView = () => { message: msg.content, agentId: msg.agentId, command: msg.agentCommand, + images: msg.images, }); } else if (msg.role === ChatMessageRole.Assistant && msg.requestId) { const request = aiChatService.sessionModel.getRequest(msg.requestId)!; diff --git a/packages/ai-native/src/browser/components/ChatEditor.tsx b/packages/ai-native/src/browser/components/ChatEditor.tsx index 4035b66e97..a4e2c7682f 100644 --- a/packages/ai-native/src/browser/components/ChatEditor.tsx +++ b/packages/ai-native/src/browser/components/ChatEditor.tsx @@ -2,6 +2,7 @@ import capitalize from 'lodash/capitalize'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import Highlight from 'react-highlight'; +import { Image } from '@opensumi/ide-components/lib/image'; import { EDITOR_COMMANDS, FILE_COMMANDS, @@ -336,6 +337,7 @@ export const CodeBlockWrapper = ({ export const CodeBlockWrapperInput = ({ text, + images, relationId, agentId, command, @@ -344,6 +346,7 @@ export const CodeBlockWrapperInput = ({ commandService, }: { text: string; + images?: string[]; relationId: string; agentId?: string; command?: string; @@ -370,6 +373,11 @@ export const CodeBlockWrapperInput = ({ return (
+ {images?.map((image) => ( +
+ +
+ ))}
{tag && (
diff --git a/packages/ai-native/src/browser/components/ChatInput.tsx b/packages/ai-native/src/browser/components/ChatInput.tsx index 7490ae3d0a..fbae384446 100644 --- a/packages/ai-native/src/browser/components/ChatInput.tsx +++ b/packages/ai-native/src/browser/components/ChatInput.tsx @@ -162,7 +162,7 @@ const AgentWidget = ({ agentId, command }) => ( ); export interface IChatInputProps { - onSend: (value: string, agentId?: string, command?: string) => void; + onSend: (value: string, images?: string[], agentId?: string, command?: string) => void; onValueChange?: (value: string) => void; onExpand?: (value: boolean) => void; placeholder?: string; @@ -341,7 +341,7 @@ export const ChatInput = React.forwardRef((props: IChatInputProps, ref) => { } const handleSendLogic = (newValue: string = value) => { - onSend(newValue, agentId, command); + onSend(newValue, [], agentId, command); setValue(''); setTheme(''); setAgentId(''); diff --git a/packages/ai-native/src/browser/components/ChatMentionInput.tsx b/packages/ai-native/src/browser/components/ChatMentionInput.tsx index 43e4720a9a..3330000478 100644 --- a/packages/ai-native/src/browser/components/ChatMentionInput.tsx +++ b/packages/ai-native/src/browser/components/ChatMentionInput.tsx @@ -1,14 +1,18 @@ +import { DataContent } from 'ai'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import { Image } from '@opensumi/ide-components/lib/image'; import { LabelService, RecentFilesManager, useInjectable } from '@opensumi/ide-core-browser'; -import { getIcon } from '@opensumi/ide-core-browser/lib/components'; -import { URI, localize } from '@opensumi/ide-core-common'; +import { Icon, getIcon } from '@opensumi/ide-core-browser/lib/components'; +import { ChatFeatureRegistryToken, URI, localize } from '@opensumi/ide-core-common'; import { CommandService } from '@opensumi/ide-core-common/lib/command'; import { WorkbenchEditorService } from '@opensumi/ide-editor'; import { FileSearchServicePath, IFileSearchService } from '@opensumi/ide-file-search'; +import { IMessageService } from '@opensumi/ide-overlay'; import { IWorkspaceService } from '@opensumi/ide-workspace'; import { IChatInternalService } from '../../common'; +import { ChatFeatureRegistry } from '../chat/chat.feature.registry'; import { ChatInternalService } from '../chat/chat.internal.service'; import { OPEN_MCP_CONFIG_COMMAND } from '../mcp/config/mcp-config.commands'; @@ -17,7 +21,13 @@ import { MentionInput } from './mention-input/mention-input'; import { FooterButtonPosition, FooterConfig, MentionItem, MentionType } from './mention-input/types'; export interface IChatMentionInputProps { - onSend: (value: string, agentId?: string, command?: string, option?: { model: string; [key: string]: any }) => void; + onSend: ( + value: string, + images?: string[], + agentId?: string, + command?: string, + option?: { model: string; [key: string]: any }, + ) => void; onValueChange?: (value: string) => void; onExpand?: (value: boolean) => void; placeholder?: string; @@ -26,6 +36,7 @@ export interface IChatMentionInputProps { sendBtnClassName?: string; defaultHeight?: number; value?: string; + images?: Array; autoFocus?: boolean; theme?: string | null; setTheme: (theme: string | null) => void; @@ -41,6 +52,7 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => { const { onSend, disabled = false } = props; const [value, setValue] = useState(props.value || ''); + const [images, setImages] = useState(props.images || []); const aiChatService = useInjectable(IChatInternalService); const commandService = useInjectable(CommandService); const searchService = useInjectable(FileSearchServicePath); @@ -48,7 +60,8 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => { const workspaceService = useInjectable(IWorkspaceService); const editorService = useInjectable(WorkbenchEditorService); const labelService = useInjectable(LabelService); - + const messageService = useInjectable(IMessageService); + const chatFeatureRegistry = useInjectable(ChatFeatureRegistryToken); const handleShowMCPConfig = React.useCallback(() => { commandService.executeCommand(OPEN_MCP_CONFIG_COMMAND.id); }, [commandService]); @@ -239,6 +252,24 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => { onClick: handleShowMCPConfig, position: FooterButtonPosition.LEFT, }, + { + id: 'upload-image', + iconClass: 'codicon codicon-file-media', + title: 'Upload Image', + onClick: () => { + const input = document.createElement('input'); + input.type = 'file'; + input.accept = 'image/*'; + input.onchange = (e) => { + const file = (e.target as HTMLInputElement).files?.[0]; + if (file) { + handleImageUpload(file); + } + }; + input.click(); + }, + position: FooterButtonPosition.LEFT, + }, ], showModelSelector: true, }), @@ -254,13 +285,47 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => { if (disabled) { return; } - onSend(content, undefined, undefined, option); + onSend( + content, + images.map((image) => image.toString()), + undefined, + undefined, + option, + ); + setImages(props.images || []); + }, + [onSend, images, disabled], + ); + + const handleImageUpload = useCallback( + async (file: File) => { + const allowedTypes = ['image/jpeg', 'image/jpg', 'image/png', 'image/webp', 'image/gif']; + if (!allowedTypes.includes(file.type)) { + messageService.error('Only JPG, PNG, WebP and GIF images are supported'); + return; + } + + const imageUploadProvider = chatFeatureRegistry.getImageUploadProvider(); + if (!imageUploadProvider) { + messageService.error('No image upload provider found'); + return; + } + const data = await imageUploadProvider.imageUpload(file); + setImages([...images, data]); }, - [onSend, editorService, disabled], + [images], + ); + + const handleDeleteImage = useCallback( + (index: number) => { + setImages(images.filter((_, i) => i !== index)); + }, + [images], ); return (
+ {images.length > 0 && } { workspaceService={workspaceService} placeholder={localize('aiNative.chat.input.placeholder.default')} footerConfig={defaultMentionInputFooterOptions} + onImageUpload={handleImageUpload} />
); }; + +const ImagePreviewer = ({ + images, + onDelete, +}: { + images: Array; + onDelete: (index: number) => void; +}) => ( +
+
+ {images.map((image, index) => ( +
+ + +
+ ))} +
+
+); diff --git a/packages/ai-native/src/browser/components/components.module.less b/packages/ai-native/src/browser/components/components.module.less index 0f35ff7632..393f617cf8 100644 --- a/packages/ai-native/src/browser/components/components.module.less +++ b/packages/ai-native/src/browser/components/components.module.less @@ -601,3 +601,52 @@ vertical-align: middle; font-size: 12px; } + +.thumbnail_container { + display: flex; + gap: 4px; + padding: 4px 12px; + .thumbnail { + width: 36px; + height: 36px; + border-radius: 4px; + position: relative; + box-shadow: 0 0 10px 0 rgba(0, 0, 0, 0.3); + padding: 2px; + display: inline-flex; + align-items: center; + img { + width: 100%; + height: 100%; + object-fit: cover; + } + } + .delete_button { + position: absolute; + top: -5px; + right: -5px; + font-size: 12px; + padding: 2px; + border: 0; + border-radius: 50%; + transition: transform 0.2s ease-in-out; + width: 16px; + height: 16px; + display: flex; + justify-content: center; + align-items: center; + background-color: var(--badge-background); + &:hover { + cursor: pointer; + transform: scale(1.2); + } + :global(.codicon) { + font-size: 12px; + color: var(--badge-foreground); + } + } +} + +.image_wrapper { + display: inline-block; +} diff --git a/packages/ai-native/src/browser/components/mention-input/mention-input.tsx b/packages/ai-native/src/browser/components/mention-input/mention-input.tsx index d162ebeeb9..e17358afd8 100644 --- a/packages/ai-native/src/browser/components/mention-input/mention-input.tsx +++ b/packages/ai-native/src/browser/components/mention-input/mention-input.tsx @@ -18,6 +18,7 @@ export const MentionInput: React.FC = ({ loading = false, mentionKeyword = MENTION_KEYWORD, onSelectionChange, + onImageUpload, labelService, workspaceService, placeholder = 'Ask anything, @ to mention', @@ -275,6 +276,21 @@ export const MentionInput: React.FC = ({ } }; + // 处理图片粘贴事件 + const handlePaste = (e: React.ClipboardEvent) => { + const items = e.clipboardData.items; + // eslint-disable-next-line @typescript-eslint/prefer-for-of + for (let i = 0; i < items.length; i++) { + if (items[i].kind === 'file' && items[i].type.startsWith('image/')) { + const file = items[i].getAsFile(); + if (file && onImageUpload) { + e.preventDefault(); + onImageUpload(file); + } + } + } + }; + // 处理键盘事件 const handleKeyDown = (e: React.KeyboardEvent) => { // 如果按下ESC键且提及面板处于活动状态或内联搜索处于活动状态 @@ -867,7 +883,7 @@ export const MentionInput: React.FC = ({ title={button.title} > = ({ contentEditable={true} onInput={handleInput} onKeyDown={handleKeyDown} + onPaste={handlePaste} onCompositionEnd={handleCompositionEnd} />
diff --git a/packages/ai-native/src/browser/components/mention-input/types.ts b/packages/ai-native/src/browser/components/mention-input/types.ts index ab576ff0d8..5524a4042f 100644 --- a/packages/ai-native/src/browser/components/mention-input/types.ts +++ b/packages/ai-native/src/browser/components/mention-input/types.ts @@ -55,7 +55,8 @@ export enum MentionType { interface FooterButton { id: string; - icon: string; + icon?: string; + iconClass?: string; title: string; onClick?: () => void; position: FooterButtonPosition; @@ -75,6 +76,7 @@ export interface MentionInputProps { placeholder?: string; loading?: boolean; onSelectionChange?: (value: string) => void; + onImageUpload?: (file: File) => void; footerConfig?: FooterConfig; // 新增配置项 mentionKeyword?: string; labelService?: LabelService; diff --git a/packages/ai-native/src/browser/context/llm-context.service.ts b/packages/ai-native/src/browser/context/llm-context.service.ts index 85e746da1d..b7cfc9fc85 100644 --- a/packages/ai-native/src/browser/context/llm-context.service.ts +++ b/packages/ai-native/src/browser/context/llm-context.service.ts @@ -1,3 +1,5 @@ +import { DataContent } from 'ai'; + import { Autowired, Injectable } from '@opensumi/di'; import { AppConfig } from '@opensumi/ide-core-browser/lib/react-providers/config-provider'; import { WithEventBus } from '@opensumi/ide-core-common/lib/event-bus/event-decorator'; diff --git a/packages/ai-native/src/browser/model/msg-history-manager.ts b/packages/ai-native/src/browser/model/msg-history-manager.ts index 2ff5e366bf..99fa7636d9 100644 --- a/packages/ai-native/src/browser/model/msg-history-manager.ts +++ b/packages/ai-native/src/browser/model/msg-history-manager.ts @@ -62,7 +62,7 @@ export class MsgHistoryManager extends Disposable { } public addUserMessage( - message: Required>, + message: Required>, ): string { return this.doAddMessage({ ...message, diff --git a/packages/ai-native/src/browser/types.ts b/packages/ai-native/src/browser/types.ts index ffa1a4e7fe..e2691760e2 100644 --- a/packages/ai-native/src/browser/types.ts +++ b/packages/ai-native/src/browser/types.ts @@ -1,3 +1,4 @@ +import { DataContent } from 'ai'; import React from 'react'; import { ZodSchema } from 'zod'; @@ -129,6 +130,7 @@ export interface IChatSlashCommandHandler { } export interface IChatFeatureRegistry { + registerImageUploadProvider(provider: IImageUploadProvider): void; registerWelcome(content: IChatWelcomeMessageContent | React.ReactNode, sampleQuestions?: ISampleQuestions[]): void; registerSlashCommand(command: IChatSlashCommandItem, handler: IChatSlashCommandHandler): void; } @@ -140,13 +142,14 @@ export type ChatWelcomeRender = (props: { export type ChatAIRoleRender = (props: { content: string }) => React.ReactElement | React.JSX.Element; export type ChatUserRoleRender = (props: { content: string; + images?: string[]; agentId?: string; command?: string; }) => React.ReactElement | React.JSX.Element; export type ChatThinkingRender = (props: { thinkingText?: string }) => React.ReactElement | React.JSX.Element; export type ChatThinkingResultRender = (props: { thinkingResult?: string }) => React.ReactElement | React.JSX.Element; export type ChatInputRender = (props: { - onSend: (value: string, agentId?: string, command?: string) => void; + onSend: (value: string, images?: string[], agentId?: string, command?: string) => void; onValueChange?: (value: string) => void; onExpand?: (value: boolean) => void; placeholder?: string; @@ -289,6 +292,10 @@ export interface IProblemFixProviderRegistry { registerHoverFixProvider(handler: IHoverFixHandler): void; } +export interface IImageUploadProvider { + imageUpload(file: File): Promise; +} + export const AINativeCoreContribution = Symbol('AINativeCoreContribution'); export interface AINativeCoreContribution { diff --git a/packages/ai-native/src/common/index.ts b/packages/ai-native/src/common/index.ts index c231508da3..7d8c6e90f0 100644 --- a/packages/ai-native/src/common/index.ts +++ b/packages/ai-native/src/common/index.ts @@ -52,6 +52,10 @@ export interface IChatMessageStructure { * 用于 chat 面板展示 */ message: string; + /** + * 图片 + */ + images?: string[]; /** * 实际调用的 prompt */ @@ -200,6 +204,7 @@ export interface IChatAgentRequest { requestId: string; command?: string; message: string; + images?: string[]; regenerate?: boolean; } @@ -237,6 +242,7 @@ export type IChatFollowup = IChatReplyFollowup | IChatResponseCommandFollowup; export interface IChatRequestMessage { prompt: string; + images?: string[]; agentId: string; command?: string; } diff --git a/packages/ai-native/src/common/llm-context.ts b/packages/ai-native/src/common/llm-context.ts index 6e942ed0d2..00e6cfc730 100644 --- a/packages/ai-native/src/common/llm-context.ts +++ b/packages/ai-native/src/common/llm-context.ts @@ -1,3 +1,5 @@ +import { DataContent } from 'ai'; + import { Event, URI } from '@opensumi/ide-core-common/lib/utils'; export interface LLMContextService { diff --git a/packages/ai-native/src/node/base-language-model.ts b/packages/ai-native/src/node/base-language-model.ts index d9cc95d5eb..cd4702e3e6 100644 --- a/packages/ai-native/src/node/base-language-model.ts +++ b/packages/ai-native/src/node/base-language-model.ts @@ -1,4 +1,13 @@ -import { CoreMessage, ToolExecutionOptions, jsonSchema, streamText, tool } from 'ai'; +import { + CoreMessage, + CoreUserMessage, + ImagePart, + TextPart, + ToolExecutionOptions, + jsonSchema, + streamText, + tool, +} from 'ai'; import { Autowired, Injectable } from '@opensumi/di'; import { IAIBackServiceOption } from '@opensumi/ide-core-common'; @@ -48,6 +57,7 @@ export abstract class BaseLanguageModel { options.trimTexts, options.system, options.maxTokens, + options.images, cancellationToken, ); } @@ -77,6 +87,7 @@ export abstract class BaseLanguageModel { trimTexts?: [string, string], systemPrompt?: string, maxTokens?: number, + images?: string[], cancellationToken?: CancellationToken, ): Promise { try { @@ -89,7 +100,18 @@ export abstract class BaseLanguageModel { }); } - const messages: CoreMessage[] = [...history, { role: 'user', content: request }]; + const messages: CoreMessage[] = [ + ...history, + { + role: 'user', + content: images?.length + ? [ + { type: 'text', text: request } as TextPart, + ...images.map((image) => ({ type: 'image', image: new URL(image) } as ImagePart)), + ] + : request, + }, + ]; const modelInfo = modelId ? this.getModelInfo(modelId) : undefined; const stream = streamText({ model: this.getModelIdentifier(provider, modelId), @@ -101,9 +123,9 @@ export abstract class BaseLanguageModel { maxTokens, temperature: modelInfo?.temperature || 0, topP: modelInfo?.topP || 0.8, - topK: modelInfo?.topK || 1, system: systemPrompt, providerOptions, + ...(!images?.length && { topK: modelInfo?.topK || 1 }), }); // 状态跟踪变量 diff --git a/packages/components/package.json b/packages/components/package.json index 8c2336e410..e1b66dd8f8 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -26,6 +26,7 @@ "raf": "^3.4.1", "rc-dialog": "^9.6.0", "rc-dropdown": "~2.4.1", + "rc-image": "^7.11.0", "rc-input-number": "^9.3.0", "rc-menu": "^9.16.0", "rc-notification": "~3.3.1", diff --git a/packages/components/src/image/index.tsx b/packages/components/src/image/index.tsx new file mode 100644 index 0000000000..467951d46d --- /dev/null +++ b/packages/components/src/image/index.tsx @@ -0,0 +1,24 @@ +import RcImage, { ImageProps as RcImageProps } from 'rc-image'; +import React from 'react'; + +import './styles.less'; +import { Icon } from '../icon'; + +export interface ImageProps extends RcImageProps { + className?: string; +} + +export const Image = (props: ImageProps) => ( + document.getElementById('main') || document.getElementsByTagName('body')?.[0], + mask: ( +
+ +
+ ), + }} + {...props} + /> +); diff --git a/packages/components/src/image/styles.less b/packages/components/src/image/styles.less new file mode 100644 index 0000000000..47cd89db65 --- /dev/null +++ b/packages/components/src/image/styles.less @@ -0,0 +1,205 @@ +@import '../style/variable.less'; +@import '../style/mixins.less'; + +@image-prefix-cls: ~'kt-image'; +@image-preview-prefix-cls: ~'@{image-prefix-cls}-preview'; + +.@{image-prefix-cls} { + position: relative; + display: inline-block; + + &-img { + width: 100%; + height: auto; + vertical-align: middle; + + &-placeholder { + background-color: @image-bg; + background-image: url('data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTYiIGhlaWdodD0iMTYiIHZpZXdCb3g9IjAgMCAxNiAxNiIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48cGF0aCBkPSJNMTQuNSAyLjVoLTEzQS41LjUgMCAwIDAgMSAzdjEwYS41LjUgMCAwIDAgLjUuNWgxM2EuNS41IDAgMCAwIC41LS41VjNhLjUuNSAwIDAgMC0uNS0uNXpNNS4yODEgNC43NWExIDEgMCAwIDEgMCAyIDEgMSAwIDAgMSAwLTJ6bTguMDMgNi44M2EuMTI3LjEyNyAwIDAgMS0uMDgxLjAzSDIuNzY5YS4xMjUuMTI1IDAgMCAxLS4wOTYtLjIwN2wyLjY2MS0zLjE1NmEuMTI2LjEyNiAwIDAgMSAuMTc3LS4wMTZsLjAxNi4wMTZMNy4wOCAxMC4wOWwyLjQ3LTIuOTNhLjEyNi4xMjYgMCAwIDEgLjE3Ny0uMDE2bC4wMTUuMDE2IDMuNTg4IDQuMjQ0YS4xMjcuMTI3IDAgMCAxLS4wMi4xNzV6IiBmaWxsPSIjOEM4QzhDIiBmaWxsLXJ1bGU9Im5vbnplcm8iLz48L3N2Zz4='); + background-repeat: no-repeat; + background-position: center center; + background-size: 30%; + } + } + + &-mask { + position: absolute; + top: 0; + right: 0; + bottom: 0; + left: 0; + display: flex; + align-items: center; + justify-content: center; + color: @text-color-inverse; + background: fade(@black, 50%); + cursor: pointer; + opacity: 0; + transition: opacity @animation-duration-slow; + + &-info { + padding: 0 @padding-xss; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + .@{iconfont-css-prefix} { + margin-inline-end: @margin-xss; + } + } + + &:hover { + opacity: 1; + } + } + + &-placeholder { + .box(); + } + + &-preview { + .modal-mask(); + + height: 100%; + text-align: center; + + &-body { + .box(); + overflow: hidden; + } + + &-img { + max-width: 100%; + max-height: 100%; + vertical-align: middle; + transform: scale3d(1, 1, 1); + cursor: grab; + transition: transform 0.3s @ease-out 0s; + user-select: none; + pointer-events: auto; + + &-wrapper { + .box(); + transition: transform 0.3s @ease-out 0s; + + &::before { + display: inline-block; + width: 1px; + height: 50%; + margin-right: -1px; + content: ''; + } + } + } + + &-moving { + .@{image-prefix-cls}-preview-img { + cursor: grabbing; + + &-wrapper { + transition-duration: 0s; + } + } + } + + &-wrap { + z-index: @zindex-image; + } + + // 暂时不需要控制 + &-operations-wrapper { + display: none; + // position: fixed; + // top: 0; + // right: 0; + // z-index: @zindex-image + 1; + // width: 100%; + } + + &-operations { + .reset-component(); + display: flex; + flex-direction: row-reverse; + align-items: center; + color: @image-preview-operation-color; + list-style: none; + background: fade(@modal-mask-bg, 10%); + pointer-events: auto; + + &-operation { + margin-left: @control-padding-horizontal; + padding: @control-padding-horizontal; + cursor: pointer; + transition: all 0.3s; + + &:hover { + background: fade(@modal-mask-bg, 20%); + } + + &-disabled { + color: @image-preview-operation-disabled-color; + pointer-events: none; + } + + &:last-of-type { + margin-left: 0; + } + } + + &-progress { + position: absolute; + left: 50%; + transform: translateX(-50%); + } + + &-icon { + font-size: @image-preview-operation-size; + } + } + + &-switch-left, + &-switch-right { + position: fixed; + top: 50%; + right: 8px; + z-index: @zindex-image + 1; + display: flex; + align-items: center; + justify-content: center; + width: 44px; + height: 44px; + color: @image-preview-operation-color; + background: fade(@modal-mask-bg, 10%); + border-radius: 50%; + transform: translateY(-50%); + cursor: pointer; + transition: all 0.3s; + pointer-events: auto; + + &:hover { + background: fade(@modal-mask-bg, 20%); + } + + &-disabled, + &-disabled:hover { + color: @image-preview-operation-disabled-color; + background: fade(@modal-mask-bg, 10%); + cursor: not-allowed; + > .@{iconfont-css-prefix} { + cursor: not-allowed; + } + } + + > .@{iconfont-css-prefix} { + font-size: 18px; + } + } + + &-switch-left { + left: 8px; + } + + &-switch-right { + right: 8px; + } + } +} diff --git a/packages/components/src/index.ts b/packages/components/src/index.ts index 343e4adde5..28df770f92 100644 --- a/packages/components/src/index.ts +++ b/packages/components/src/index.ts @@ -9,6 +9,7 @@ export * from './tabs'; export * from './overlay'; export * from './dialog'; export * from './icon'; +export * from './image'; export * from './notification'; export * from './popover'; export * from './message'; diff --git a/packages/components/src/style/mixins.less b/packages/components/src/style/mixins.less index 6f2c91b635..e082f796e2 100644 --- a/packages/components/src/style/mixins.less +++ b/packages/components/src/style/mixins.less @@ -133,3 +133,40 @@ clear: both; } } + +.box(@position: absolute) { + position: @position; + top: 0; + right: 0; + bottom: 0; + left: 0; +} + +.modal-mask() { + pointer-events: none; + + &.@{prefix}-zoom-enter, + &.@{prefix}-zoom-appear { + transform: none; // reset scale avoid mousePosition bug + opacity: 0; + animation-duration: @animation-duration-slow; + user-select: none; // https://github.com/ant-design/ant-design/issues/11777 + } + + &-mask { + .box(fixed); + z-index: @zindex-modal-mask; + height: 100%; + background-color: @modal-mask-bg; + + &-hidden { + display: none; + } + } + + &-wrap { + .box(fixed); + overflow: auto; + outline: 0; + } +} diff --git a/packages/components/src/style/variable.less b/packages/components/src/style/variable.less index 46ff8f92fe..c3e8385da5 100644 --- a/packages/components/src/style/variable.less +++ b/packages/components/src/style/variable.less @@ -59,6 +59,7 @@ @zindex-dropdown: 1050; @zindex-picker: 1050; @zindex-tooltip: 1060; +@zindex-image: 1080; // Shadow @shadow-color: rgba(0, 0, 0, 0.15); @@ -112,6 +113,14 @@ @padding-md: 16px; // small containers and buttons @padding-sm: 12px; // Form controls and items @padding-xs: 8px; // small items +@padding-xss: 4px; // more small + +// vertical margins +@margin-lg: 24px; // containers +@margin-md: 16px; // small containers and buttons +@margin-sm: 12px; // Form controls and items +@margin-xs: 8px; // small items +@margin-xss: 4px; // more small // vertical padding for all form controls @control-padding-horizontal: @padding-sm; @@ -205,3 +214,13 @@ // Checkbox @checkbox-default-size: 12px; @checkbox-large-size: 14px; + +// Image +@image-size-base: 48px; +@image-font-size-base: 24px; +@image-bg: #f5f5f5; +@image-color: #fff; +@image-mask-font-size: 16px; +@image-preview-operation-size: 18px; +@image-preview-operation-color: @text-color-dark; +@image-preview-operation-disabled-color: fade(@image-preview-operation-color, 25%); diff --git a/packages/core-browser/src/components/ai-native/enhanceIcon/styles.module.less b/packages/core-browser/src/components/ai-native/enhanceIcon/styles.module.less index 726230dd56..2d8c01e90a 100644 --- a/packages/core-browser/src/components/ai-native/enhanceIcon/styles.module.less +++ b/packages/core-browser/src/components/ai-native/enhanceIcon/styles.module.less @@ -4,7 +4,6 @@ display: flex; align-items: center; cursor: pointer; - text-wrap: nowrap; white-space: nowrap; box-sizing: border-box; diff --git a/packages/core-common/src/types/ai-native/index.ts b/packages/core-common/src/types/ai-native/index.ts index 4bb1bc1aa4..66ac85a138 100644 --- a/packages/core-common/src/types/ai-native/index.ts +++ b/packages/core-common/src/types/ai-native/index.ts @@ -171,6 +171,7 @@ export interface IAIBackServiceOption { type?: string; requestId?: string; sessionId?: string; + images?: string[]; history?: CoreMessage[]; tools?: any[]; clientId?: string; @@ -440,6 +441,7 @@ export interface IHistoryChatMessage extends IChatMessage { order: number; type?: 'string' | 'component'; + images?: string[]; relationId?: string; componentId?: string; componentValue?: any; diff --git a/packages/startup/entry/sample-modules/ai-native/ai-native.contribution.ts b/packages/startup/entry/sample-modules/ai-native/ai-native.contribution.ts index 8e63f9ab71..ad3b80a750 100644 --- a/packages/startup/entry/sample-modules/ai-native/ai-native.contribution.ts +++ b/packages/startup/entry/sample-modules/ai-native/ai-native.contribution.ts @@ -20,11 +20,9 @@ import { IRenameCandidatesProviderRegistry, IResolveConflictRegistry, ITerminalProviderRegistry, - TChatSlashCommandSend, TerminalSuggestionReadableStream, } from '@opensumi/ide-ai-native/lib/browser/types'; import { InlineChatController } from '@opensumi/ide-ai-native/lib/browser/widget/inline-chat/inline-chat-controller'; -import { SerializedContext } from '@opensumi/ide-ai-native/lib/common/llm-context'; import { ChatAgentPromptProvider, DefaultChatAgentPromptProvider, @@ -48,8 +46,6 @@ import { import { ICodeEditor, ISelection, NewSymbolName, NewSymbolNameTag, Range, Selection } from '@opensumi/ide-monaco'; import { MarkdownString } from '@opensumi/monaco-editor-core/esm/vs/base/common/htmlContent'; -import { SlashCommand } from './SlashCommand'; - export enum EInlineOperation { Comments = 'Comments', Optimize = 'Optimize', @@ -236,6 +232,10 @@ export class AINativeContribution implements AINativeCoreContribution { ], ); + registry.registerImageUploadProvider({ + imageUpload: imageToBase64, + }); + // registry.registerSlashCommand( // { // name: 'Explain', @@ -509,3 +509,23 @@ export class AINativeContribution implements AINativeCoreContribution { }); } } + +const MAX_IMAGE_SIZE = 3 * 1024 * 1024; + +const imageToBase64 = (file: File) => + new Promise((resolve, reject) => { + if (file.size > MAX_IMAGE_SIZE) { + reject(new Error('Image size exceeds 3MB limit')); + return; + } + + const reader = new FileReader(); + reader.onload = () => { + const base64String = reader.result as string; + resolve(base64String); + }; + reader.onerror = () => { + reject(new Error('Failed to convert image to base64')); + }; + reader.readAsDataURL(file); + }); diff --git a/yarn.lock b/yarn.lock index f06e062f4a..030082b716 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3490,6 +3490,7 @@ __metadata: raf: "npm:^3.4.1" rc-dialog: "npm:^9.6.0" rc-dropdown: "npm:~2.4.1" + rc-image: "npm:^7.11.0" rc-input-number: "npm:^9.3.0" rc-menu: "npm:^9.16.0" rc-notification: "npm:~3.3.1" @@ -20350,7 +20351,7 @@ __metadata: languageName: node linkType: hard -"rc-image@npm:~7.11.0": +"rc-image@npm:^7.11.0, rc-image@npm:~7.11.0": version: 7.11.0 resolution: "rc-image@npm:7.11.0" dependencies: