From 5042b3b1d44d56bd8f9c668721f8af443ac7fa0b Mon Sep 17 00:00:00 2001 From: CH Date: Thu, 6 Feb 2025 20:50:07 +0800 Subject: [PATCH 01/83] feat(canvas): Add image node drag and drop support - Implement drag and drop functionality for adding images to canvas - Add image node type with preview and metadata support - Update node types, colors, and icons to include image nodes - Extend schema to support image node type --- .../src/components/canvas/index.tsx | 48 +++- .../launchpad/context-manager/utils/icon.tsx | 3 + .../canvas/node-action-menu/index.tsx | 1 + .../components/canvas/node-preview/index.tsx | 10 + .../src/components/canvas/nodes/image.tsx | 219 ++++++++++++++++++ .../src/components/canvas/nodes/index.ts | 8 + .../components/canvas/nodes/shared/colors.ts | 1 + .../components/canvas/nodes/shared/types.ts | 5 + .../src/components/common/icon.tsx | 4 + packages/openapi-schema/schema.yml | 1 + 10 files changed, 298 insertions(+), 2 deletions(-) create mode 100644 packages/ai-workspace-common/src/components/canvas/nodes/image.tsx diff --git a/packages/ai-workspace-common/src/components/canvas/index.tsx b/packages/ai-workspace-common/src/components/canvas/index.tsx index a5e8f7226..5824e39e6 100644 --- a/packages/ai-workspace-common/src/components/canvas/index.tsx +++ b/packages/ai-workspace-common/src/components/canvas/index.tsx @@ -487,6 +487,50 @@ const Flow = memo(({ canvasId }: { canvasId: string }) => { [reactFlowInstance], ); + // Add drag and drop handlers + const handleDragOver = useCallback((event: React.DragEvent) => { + event.preventDefault(); + event.dataTransfer.dropEffect = 'copy'; + }, []); + + const handleDrop = useCallback( + (event: React.DragEvent) => { + event.preventDefault(); + + const files = Array.from(event.dataTransfer.files); + const imageFile = files.find((file) => file.type.startsWith('image/')); + + if (imageFile) { + const reader = new FileReader(); + reader.onload = (e) => { + const imageUrl = e.target?.result as string; + + // Get drop position in flow coordinates + const flowPosition = reactFlowInstance.screenToFlowPosition({ + x: event.clientX, + y: event.clientY, + }); + + // Add image node + addNode({ + type: 'image', + data: { + title: imageFile.name, + entityId: `image-${Date.now()}`, + metadata: { + imageUrl, + sizeMode: 'adaptive', + }, + }, + position: flowPosition, + }); + }; + reader.readAsDataURL(imageFile); + } + }, + [addNode, reactFlowInstance], + ); + return ( { onNodeDragStop={onNodeDragStop} nodeDragThreshold={10} nodesDraggable={!operatingNodeId} - // onlyRenderVisibleElements={true} - elevateNodesOnSelect={false} onSelectionContextMenu={onSelectionContextMenu} deleteKeyCode={['Backspace', 'Delete']} multiSelectionKeyCode={['Shift', 'Meta']} + onDragOver={handleDragOver} + onDrop={handleDrop} > {nodes?.length === 0 && hasCanvasSynced && (
diff --git a/packages/ai-workspace-common/src/components/canvas/launchpad/context-manager/utils/icon.tsx b/packages/ai-workspace-common/src/components/canvas/launchpad/context-manager/utils/icon.tsx index bf7f54588..d02b1ac7e 100644 --- a/packages/ai-workspace-common/src/components/canvas/launchpad/context-manager/utils/icon.tsx +++ b/packages/ai-workspace-common/src/components/canvas/launchpad/context-manager/utils/icon.tsx @@ -5,6 +5,7 @@ import { IconDocumentFilled, IconThreadHistoryFilled, IconQuote, + IconImageFilled, } from '@refly-packages/ai-workspace-common/components/common/icon'; import { NODE_COLORS } from '@refly-packages/ai-workspace-common/components/canvas/nodes/shared/colors'; import { CanvasNodeType, SelectionKey } from '@refly/openapi-schema'; @@ -33,6 +34,8 @@ export const getContextItemIcon = ( case 'documentSelection': case 'skillResponseSelection': return ; + case 'image': + return ; default: return null; } diff --git a/packages/ai-workspace-common/src/components/canvas/node-action-menu/index.tsx b/packages/ai-workspace-common/src/components/canvas/node-action-menu/index.tsx index dc3bfac5f..d3834246d 100644 --- a/packages/ai-workspace-common/src/components/canvas/node-action-menu/index.tsx +++ b/packages/ai-workspace-common/src/components/canvas/node-action-menu/index.tsx @@ -431,6 +431,7 @@ export const NodeActionMenu: FC = ({ }, }, ], + image: [], memo: [ { key: 'insertToDoc', diff --git a/packages/ai-workspace-common/src/components/canvas/node-preview/index.tsx b/packages/ai-workspace-common/src/components/canvas/node-preview/index.tsx index 79ffa2fbd..0242a683e 100644 --- a/packages/ai-workspace-common/src/components/canvas/node-preview/index.tsx +++ b/packages/ai-workspace-common/src/components/canvas/node-preview/index.tsx @@ -31,6 +31,16 @@ const PreviewComponent = memo( return ; case 'skillResponse': return ; + case 'image': + return ( +
+ {node.data?.title +
+ ); default: return null; } diff --git a/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx b/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx new file mode 100644 index 000000000..1fd55b1e5 --- /dev/null +++ b/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx @@ -0,0 +1,219 @@ +import React, { memo, useState, useCallback, useRef, useMemo, useEffect } from 'react'; +import { NodeProps, useReactFlow, Position } from '@xyflow/react'; +import { CanvasNode, CommonNodeProps } from './shared/types'; +import { ActionButtons } from './shared/action-buttons'; +import { useNodeHoverEffect } from '@refly-packages/ai-workspace-common/hooks/canvas/use-node-hover'; +import { useNodeSize } from '@refly-packages/ai-workspace-common/hooks/canvas/use-node-size'; +import { NodeResizer as NodeResizerComponent } from './shared/node-resizer'; +import { useCanvasStoreShallow } from '@refly-packages/ai-workspace-common/stores/canvas'; +import { getNodeCommonStyles } from './index'; +import { CustomHandle } from './shared/custom-handle'; +import classNames from 'classnames'; +import { NodeHeader } from './shared/node-header'; +import { IconImage } from '@refly-packages/ai-workspace-common/components/common/icon'; +import { + nodeActionEmitter, + createNodeEventName, + cleanupNodeEvents, +} from '@refly-packages/ai-workspace-common/events/nodeActions'; +import { useAddNode } from '@refly-packages/ai-workspace-common/hooks/canvas/use-add-node'; +import { genSkillID } from '@refly-packages/utils/id'; +import { IContextItem } from '@refly-packages/ai-workspace-common/stores/context-panel'; +import { useAddToContext } from '@refly-packages/ai-workspace-common/hooks/canvas/use-add-to-context'; +import { useDeleteNode } from '@refly-packages/ai-workspace-common/hooks/canvas/use-delete-node'; + +// Define image node metadata type +export interface ImageNodeMeta { + imageUrl: string; + sizeMode?: 'compact' | 'adaptive'; + style?: React.CSSProperties; + originalWidth?: number; +} + +// Define image node props type +export type ImageNodeProps = NodeProps> & CommonNodeProps; + +export const ImageNode = memo( + ({ id, data, isPreview, selected, hideActions, hideHandles, onNodeClick }: ImageNodeProps) => { + const { metadata } = data ?? {}; + const imageUrl = metadata?.imageUrl; + const [isHovered, setIsHovered] = useState(false); + const { handleMouseEnter: onHoverStart, handleMouseLeave: onHoverEnd } = useNodeHoverEffect(id); + const targetRef = useRef(null); + const { getNode } = useReactFlow(); + const { addNode } = useAddNode(); + const { addToContext } = useAddToContext(); + const { deleteNode } = useDeleteNode(); + + const { operatingNodeId } = useCanvasStoreShallow((state) => ({ + operatingNodeId: state.operatingNodeId, + })); + + const isOperating = operatingNodeId === id; + const sizeMode = metadata?.sizeMode || 'adaptive'; + const node = useMemo(() => getNode(id), [id, getNode]); + + const { containerStyle, handleResize } = useNodeSize({ + id, + node, + sizeMode, + isOperating, + minWidth: 100, + maxWidth: 800, + minHeight: 80, + defaultWidth: 288, + defaultHeight: 384, + }); + + const handleMouseEnter = useCallback(() => { + setIsHovered(true); + onHoverStart(); + }, [onHoverStart]); + + const handleMouseLeave = useCallback(() => { + setIsHovered(false); + onHoverEnd(); + }, [onHoverEnd]); + + const handleAddToContext = useCallback(() => { + addToContext({ + type: 'image', + title: data.title, + entityId: data.entityId, + metadata: data.metadata, + }); + }, [data, addToContext]); + + const handleDelete = useCallback(() => { + deleteNode({ + id, + type: 'image', + data, + position: { x: 0, y: 0 }, + } as unknown as CanvasNode); + }, [id, data, deleteNode]); + + const handleAskAI = useCallback(() => { + addNode( + { + type: 'skill', + data: { + title: 'Skill', + entityId: genSkillID(), + metadata: { + contextItems: [ + { + type: 'image', + title: data.title, + entityId: data.entityId, + metadata: data.metadata, + }, + ] as IContextItem[], + }, + }, + }, + [{ type: 'image', entityId: data.entityId }], + false, + true, + ); + }, [data, addNode]); + + // Add event handling + useEffect(() => { + // Create node-specific event handlers + const handleNodeAddToContext = () => handleAddToContext(); + const handleNodeDelete = () => handleDelete(); + const handleNodeAskAI = () => handleAskAI(); + + // Register events with node ID + nodeActionEmitter.on(createNodeEventName(id, 'addToContext'), handleNodeAddToContext); + nodeActionEmitter.on(createNodeEventName(id, 'delete'), handleNodeDelete); + nodeActionEmitter.on(createNodeEventName(id, 'askAI'), handleNodeAskAI); + + return () => { + // Cleanup events when component unmounts + nodeActionEmitter.off(createNodeEventName(id, 'addToContext'), handleNodeAddToContext); + nodeActionEmitter.off(createNodeEventName(id, 'delete'), handleNodeDelete); + nodeActionEmitter.off(createNodeEventName(id, 'askAI'), handleNodeAskAI); + + // Clean up all node events + cleanupNodeEvents(id); + }; + }, [id, handleAddToContext, handleDelete, handleAskAI]); + + if (!data || !imageUrl) { + return null; + } + + return ( +
+
+ {!isPreview && !hideActions && ( + + )} + +
+ {!isPreview && !hideHandles && ( + <> + + + + )} + +
+ + +
+ {data.title +
+
+
+
+ + {!isPreview && selected && sizeMode === 'adaptive' && ( + + )} +
+ ); + }, +); + +ImageNode.displayName = 'ImageNode'; diff --git a/packages/ai-workspace-common/src/components/canvas/nodes/index.ts b/packages/ai-workspace-common/src/components/canvas/nodes/index.ts index b384a39ca..53d432b4a 100644 --- a/packages/ai-workspace-common/src/components/canvas/nodes/index.ts +++ b/packages/ai-workspace-common/src/components/canvas/nodes/index.ts @@ -7,6 +7,7 @@ import { ToolNode } from './tool'; import { SkillResponseNode } from './skill-response'; import { MemoNode } from './memo/memo'; import { GroupNode } from './group'; +import { ImageNode } from './image'; import { NodeMetadataMap, CanvasNodeData, @@ -38,6 +39,7 @@ export const nodeTypes: NodeTypes = { skillResponse: SkillResponseNode, memo: MemoNode, group: GroupNode, + image: ImageNode, }; // Helper function to prepare node data @@ -128,6 +130,12 @@ export const getNodeDefaultMetadata = (nodeType: CanvasNodeType) => { executionTime: null, } as ResponseNodeMeta; + case 'image': + return { + sizeMode: 'adaptive', + style: {}, + }; + default: return {}; } diff --git a/packages/ai-workspace-common/src/components/canvas/nodes/shared/colors.ts b/packages/ai-workspace-common/src/components/canvas/nodes/shared/colors.ts index 51cffddf0..e5e07548e 100644 --- a/packages/ai-workspace-common/src/components/canvas/nodes/shared/colors.ts +++ b/packages/ai-workspace-common/src/components/canvas/nodes/shared/colors.ts @@ -11,4 +11,5 @@ export const NODE_COLORS: Record = { memo: '#6172F3', group: '#6172F3', threadHistory: '#64748b', + image: '#02b0c7', }; diff --git a/packages/ai-workspace-common/src/components/canvas/nodes/shared/types.ts b/packages/ai-workspace-common/src/components/canvas/nodes/shared/types.ts index 1c1a346b8..a9481f483 100644 --- a/packages/ai-workspace-common/src/components/canvas/nodes/shared/types.ts +++ b/packages/ai-workspace-common/src/components/canvas/nodes/shared/types.ts @@ -71,6 +71,10 @@ export type ResponseNodeMeta = { originalWidth?: number; }; +export type ImageNodeMeta = { + imageType: string; +}; + // Type mapping for node metadata export type NodeMetadataMap = { document: DocumentNodeMeta; @@ -78,6 +82,7 @@ export type NodeMetadataMap = { skill: SkillNodeMeta; tool: ToolNodeMeta; response: ResponseNodeMeta; + image: ImageNodeMeta; } & Record>; // Add new common props interface diff --git a/packages/ai-workspace-common/src/components/common/icon.tsx b/packages/ai-workspace-common/src/components/common/icon.tsx index eab0f87ca..4824047e0 100644 --- a/packages/ai-workspace-common/src/components/common/icon.tsx +++ b/packages/ai-workspace-common/src/components/common/icon.tsx @@ -40,6 +40,8 @@ import { RiChatHistoryLine, RiChatHistoryFill, RiUploadCloud2Line, + RiImageAiFill, + RiImageAiLine, } from 'react-icons/ri'; import { TiDocumentDelete } from 'react-icons/ti'; import { AiOutlineLoading3Quarters } from 'react-icons/ai'; @@ -73,6 +75,8 @@ import { memo } from 'react'; export const IconCanvas = TfiBlackboard; export const IconAskAI = LuSparkles; export const IconAskAIInput = TbInputSpark; +export const IconImage = RiImageAiLine; +export const IconImageFilled = RiImageAiFill; export const IconDocument = LuFileText; export const IconDocumentFilled = HiDocumentText; export const IconCreateDocument = LuFilePlus; diff --git a/packages/openapi-schema/schema.yml b/packages/openapi-schema/schema.yml index d01b818ce..b90ee0f44 100644 --- a/packages/openapi-schema/schema.yml +++ b/packages/openapi-schema/schema.yml @@ -4462,6 +4462,7 @@ components: - toolResponse - memo - group + - image CanvasNodeData: type: object required: From 3d805777c0c2dfa8f9cbb5f040325e35670be5a5 Mon Sep 17 00:00:00 2001 From: CH Date: Thu, 6 Feb 2025 22:05:25 +0800 Subject: [PATCH 02/83] feat(canvas): Enhance image node preview and handling - Add image preview modal using Ant Design Image component - Implement dynamic image node resizing - Update node preview logic to handle image nodes - Add image node type to translations - Generate unique image IDs using utility function --- .../src/components/canvas/index.tsx | 5 +- .../canvas/node-action-menu/index.tsx | 14 +++-- .../components/canvas/node-preview/index.tsx | 10 ---- .../src/components/canvas/nodes/image.tsx | 51 ++++++++++++++++--- packages/i18n/src/en-US/ui.ts | 1 + packages/i18n/src/zh-Hans/ui.ts | 1 + packages/utils/src/id.ts | 5 ++ 7 files changed, 64 insertions(+), 23 deletions(-) diff --git a/packages/ai-workspace-common/src/components/canvas/index.tsx b/packages/ai-workspace-common/src/components/canvas/index.tsx index 5824e39e6..c71d9bdbd 100644 --- a/packages/ai-workspace-common/src/components/canvas/index.tsx +++ b/packages/ai-workspace-common/src/components/canvas/index.tsx @@ -13,6 +13,7 @@ import { useCreateDocument } from '@refly-packages/ai-workspace-common/hooks/can import { useNodeOperations } from '@refly-packages/ai-workspace-common/hooks/canvas/use-node-operations'; import { useNodeSelection } from '@refly-packages/ai-workspace-common/hooks/canvas/use-node-selection'; import { useAddNode } from '@refly-packages/ai-workspace-common/hooks/canvas/use-add-node'; +import { genImageID } from '@refly-packages/utils/id'; import { CanvasProvider, useCanvasContext, @@ -407,7 +408,7 @@ const Flow = memo(({ canvasId }: { canvasId: string }) => { } // Memo nodes are not previewable - if (node.type === 'memo' || node.type === 'skill' || node.type === 'group') { + if (['memo', 'skill', 'group', 'image'].includes(node.type)) { return; } @@ -516,7 +517,7 @@ const Flow = memo(({ canvasId }: { canvasId: string }) => { type: 'image', data: { title: imageFile.name, - entityId: `image-${Date.now()}`, + entityId: genImageID(), metadata: { imageUrl, sizeMode: 'adaptive', diff --git a/packages/ai-workspace-common/src/components/canvas/node-action-menu/index.tsx b/packages/ai-workspace-common/src/components/canvas/node-action-menu/index.tsx index d3834246d..7daa28399 100644 --- a/packages/ai-workspace-common/src/components/canvas/node-action-menu/index.tsx +++ b/packages/ai-workspace-common/src/components/canvas/node-action-menu/index.tsx @@ -168,11 +168,15 @@ export const NodeActionMenu: FC = ({ }, [nodeId, nodeData?.contentPreview, onClose]); const handlePreview = useCallback(() => { - addNodePreview(canvasId, node); - locateToNodePreviewEmitter.emit('locateToNodePreview', { - id: nodeId, - canvasId, - }); + if (nodeType === 'image') { + nodeActionEmitter.emit(createNodeEventName(nodeId, 'preview')); + } else { + addNodePreview(canvasId, node); + locateToNodePreviewEmitter.emit('locateToNodePreview', { + id: nodeId, + canvasId, + }); + } onClose?.(); }, [node, nodeId, canvasId, onClose, addNodePreview]); diff --git a/packages/ai-workspace-common/src/components/canvas/node-preview/index.tsx b/packages/ai-workspace-common/src/components/canvas/node-preview/index.tsx index 0242a683e..79ffa2fbd 100644 --- a/packages/ai-workspace-common/src/components/canvas/node-preview/index.tsx +++ b/packages/ai-workspace-common/src/components/canvas/node-preview/index.tsx @@ -31,16 +31,6 @@ const PreviewComponent = memo( return ; case 'skillResponse': return ; - case 'image': - return ( -
- {node.data?.title -
- ); default: return null; } diff --git a/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx b/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx index 1fd55b1e5..927823d1c 100644 --- a/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx +++ b/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx @@ -1,5 +1,6 @@ import React, { memo, useState, useCallback, useRef, useMemo, useEffect } from 'react'; import { NodeProps, useReactFlow, Position } from '@xyflow/react'; +import { Image } from 'antd'; import { CanvasNode, CommonNodeProps } from './shared/types'; import { ActionButtons } from './shared/action-buttons'; import { useNodeHoverEffect } from '@refly-packages/ai-workspace-common/hooks/canvas/use-node-hover'; @@ -21,6 +22,7 @@ import { genSkillID } from '@refly-packages/utils/id'; import { IContextItem } from '@refly-packages/ai-workspace-common/stores/context-panel'; import { useAddToContext } from '@refly-packages/ai-workspace-common/hooks/canvas/use-add-to-context'; import { useDeleteNode } from '@refly-packages/ai-workspace-common/hooks/canvas/use-delete-node'; +import Moveable from 'react-moveable'; // Define image node metadata type export interface ImageNodeMeta { @@ -38,6 +40,7 @@ export const ImageNode = memo( const { metadata } = data ?? {}; const imageUrl = metadata?.imageUrl; const [isHovered, setIsHovered] = useState(false); + const [isPreviewModalVisible, setIsPreviewModalVisible] = useState(false); const { handleMouseEnter: onHoverStart, handleMouseLeave: onHoverEnd } = useNodeHoverEffect(id); const targetRef = useRef(null); const { getNode } = useReactFlow(); @@ -62,7 +65,7 @@ export const ImageNode = memo( maxWidth: 800, minHeight: 80, defaultWidth: 288, - defaultHeight: 384, + defaultHeight: 'auto', }); const handleMouseEnter = useCallback(() => { @@ -118,28 +121,48 @@ export const ImageNode = memo( ); }, [data, addNode]); + const handlePreview = useCallback(() => { + setIsPreviewModalVisible(true); + }, []); + // Add event handling useEffect(() => { // Create node-specific event handlers const handleNodeAddToContext = () => handleAddToContext(); const handleNodeDelete = () => handleDelete(); const handleNodeAskAI = () => handleAskAI(); + const handleNodePreview = () => handlePreview(); // Register events with node ID nodeActionEmitter.on(createNodeEventName(id, 'addToContext'), handleNodeAddToContext); nodeActionEmitter.on(createNodeEventName(id, 'delete'), handleNodeDelete); nodeActionEmitter.on(createNodeEventName(id, 'askAI'), handleNodeAskAI); + nodeActionEmitter.on(createNodeEventName(id, 'preview'), handleNodePreview); return () => { // Cleanup events when component unmounts nodeActionEmitter.off(createNodeEventName(id, 'addToContext'), handleNodeAddToContext); nodeActionEmitter.off(createNodeEventName(id, 'delete'), handleNodeDelete); nodeActionEmitter.off(createNodeEventName(id, 'askAI'), handleNodeAskAI); + nodeActionEmitter.off(createNodeEventName(id, 'preview'), handleNodePreview); // Clean up all node events cleanupNodeEvents(id); }; - }, [id, handleAddToContext, handleDelete, handleAskAI]); + }, [id, handleAddToContext, handleDelete, handleAskAI, handlePreview]); + + const moveableRef = useRef(null); + + const resizeMoveable = useCallback((width: number, height: number) => { + moveableRef.current?.request('resizable', { width, height }); + }, []); + + useEffect(() => { + if (!targetRef.current) return; + + const { offsetWidth, offsetHeight } = targetRef.current; + resizeMoveable(offsetWidth, offsetHeight); + }, [resizeMoveable, targetRef.current?.offsetHeight]); if (!data || !imageUrl) { return null; @@ -163,10 +186,10 @@ export const ImageNode = memo(
{!isPreview && !hideHandles && ( <> @@ -196,6 +219,21 @@ export const ImageNode = memo( alt={data.title || 'Image'} className="w-full h-auto object-contain" /> + + {/* only for preview image */} + {isPreviewModalVisible && ( + { + setIsPreviewModalVisible(value); + }, + }} + /> + )}
@@ -203,6 +241,7 @@ export const ImageNode = memo( {!isPreview && selected && sizeMode === 'adaptive' && ( Date: Thu, 6 Feb 2025 22:25:00 +0800 Subject: [PATCH 03/83] feat(api): support passing images as skill input --- apps/api/src/action/action.dto.ts | 6 -- apps/api/src/config/app.config.ts | 1 + apps/api/src/misc/misc.service.ts | 56 +++++++++++ apps/api/src/skill/skill.module.ts | 2 + apps/api/src/skill/skill.service.ts | 6 ++ packages/ai-workspace-common/package.json | 2 +- .../ai-workspace-common/src/queries/common.ts | 14 --- .../src/queries/queries.ts | 40 -------- .../src/requests/services.gen.ts | 36 ------- .../src/requests/types.gen.ts | 85 +++------------- packages/openapi-schema/openapi-ts.config.ts | 2 +- packages/openapi-schema/schema.yml | 96 +++---------------- packages/openapi-schema/src/schemas.gen.ts | 90 +++++------------ packages/openapi-schema/src/services.gen.ts | 36 ------- packages/openapi-schema/src/types.gen.ts | 85 +++------------- packages/skill-template/src/base.ts | 4 + .../src/scheduler/utils/message.ts | 9 +- .../skill-template/src/skills/common-qna.ts | 3 +- .../skill-template/src/skills/edit-doc.ts | 18 ++-- .../skill-template/src/skills/generate-doc.ts | 3 +- .../src/skills/library-search.ts | 3 +- .../skill-template/src/skills/rewrite-doc.ts | 12 +-- .../skill-template/src/skills/web-search.ts | 3 +- 23 files changed, 155 insertions(+), 457 deletions(-) diff --git a/apps/api/src/action/action.dto.ts b/apps/api/src/action/action.dto.ts index dc0e3f70d..9e663481a 100644 --- a/apps/api/src/action/action.dto.ts +++ b/apps/api/src/action/action.dto.ts @@ -1,5 +1,4 @@ import { - InvokeActionRequest, ActionResult, ActionStep, ActionType, @@ -14,11 +13,6 @@ import { import { pick } from '@/utils'; import { modelInfoPO2DTO } from '@/misc/misc.dto'; -export interface InvokeActionJobData extends InvokeActionRequest { - uid: string; - rawParam: string; -} - export function actionStepPO2DTO(step: ActionStepModel): ActionStep { return { ...pick(step, ['name', 'content']), diff --git a/apps/api/src/config/app.config.ts b/apps/api/src/config/app.config.ts index 618b6779c..1e24f6ddc 100644 --- a/apps/api/src/config/app.config.ts +++ b/apps/api/src/config/app.config.ts @@ -17,6 +17,7 @@ export default () => ({ }, origin: process.env.ORIGIN || 'http://localhost:5700', staticEndpoint: process.env.STATIC_ENDPOINT || 'http://localhost:5800/v1/misc/', + imagePayloadMode: process.env.IMAGE_PAYLOAD_MODE || 'base64', // 'url' or 'base64' minio: { internal: { endPoint: process.env.MINIO_INTERNAL_ENDPOINT || 'localhost', diff --git a/apps/api/src/misc/misc.service.ts b/apps/api/src/misc/misc.service.ts index c3c78e8ed..6ff62f0ce 100644 --- a/apps/api/src/misc/misc.service.ts +++ b/apps/api/src/misc/misc.service.ts @@ -137,6 +137,7 @@ export class MiscService { }); return { + storageKey, url: `${this.config.get('staticEndpoint')}${storageKey}`, }; } @@ -226,4 +227,59 @@ export class MiscService { const data = await this.minio.client.getObject(`static/${objectKey}`); return new StreamableFile(data); } + + /** + * Generates image URLs based on storage keys and configured payload mode + * @param storageKeys - Array of storage keys for the images + * @returns Array of URLs (either base64 or regular URLs depending on config) + */ + async generateImageUrls(storageKeys: string[]): Promise { + if (!Array.isArray(storageKeys) || storageKeys.length === 0) { + return []; + } + + let imageMode = this.config.get('imagePayloadMode'); + if (imageMode === 'url' && !this.config.get('staticEndpoint')) { + this.logger.warn('Static endpoint is not configured, fallback to base64 mode'); + imageMode = 'base64'; + } + + this.logger.log(`Generating image URLs in ${imageMode} mode for ${storageKeys.length} images`); + + try { + if (imageMode === 'base64') { + const urls = await Promise.all( + storageKeys.map(async (key) => { + try { + const data = await this.minio.client.getObject(key); + const chunks: Buffer[] = []; + + for await (const chunk of data) { + chunks.push(chunk); + } + + const buffer = Buffer.concat(chunks); + const base64 = buffer.toString('base64'); + const contentType = await this.minio.client + .statObject(`static/${key}`) + .then((stat) => stat.metaData?.['content-type'] ?? 'image/jpeg'); + + return `data:${contentType};base64,${base64}`; + } catch (error) { + this.logger.error(`Failed to generate base64 for key ${key}:`, error); + return ''; + } + }), + ); + return urls.filter(Boolean); + } + + // URL mode + const staticEndpoint = this.config.get('staticEndpoint') ?? ''; + return storageKeys.map((key) => `${staticEndpoint}static/${key}`); + } catch (error) { + this.logger.error('Error generating image URLs:', error); + return []; + } + } } diff --git a/apps/api/src/skill/skill.module.ts b/apps/api/src/skill/skill.module.ts index 176ad8806..3ddc4a6bc 100644 --- a/apps/api/src/skill/skill.module.ts +++ b/apps/api/src/skill/skill.module.ts @@ -17,6 +17,7 @@ import { LabelModule } from '@/label/label.module'; import { SkillProcessor, SkillTimeoutCheckProcessor } from '@/skill/skill.processor'; import { SubscriptionModule } from '@/subscription/subscription.module'; import { CollabModule } from '@/collab/collab.module'; +import { MiscModule } from '@/misc/misc.module'; @Module({ imports: [ @@ -28,6 +29,7 @@ import { CollabModule } from '@/collab/collab.module'; RAGModule, SubscriptionModule, CollabModule, + MiscModule, BullModule.registerQueue({ name: QUEUE_SKILL }), BullModule.registerQueue({ name: QUEUE_SKILL_TIMEOUT_CHECK }), BullModule.registerQueue({ name: QUEUE_SYNC_TOKEN_USAGE }), diff --git a/apps/api/src/skill/skill.service.ts b/apps/api/src/skill/skill.service.ts index 7f7070306..9a61647c1 100644 --- a/apps/api/src/skill/skill.service.ts +++ b/apps/api/src/skill/skill.service.ts @@ -88,6 +88,7 @@ import { ResultAggregator } from '@/utils/result'; import { CollabContext } from '@/collab/collab.dto'; import { DirectConnection } from '@hocuspocus/server'; import { modelInfoPO2DTO } from '@/misc/misc.dto'; +import type { MiscService } from '@/misc/misc.service'; export function createLangchainMessage(result: ActionResult, steps: ActionStep[]): BaseMessage[] { const query = result.title; @@ -138,6 +139,7 @@ export class SkillService { private canvas: CanvasService, private subscription: SubscriptionService, private collabService: CollabService, + private misc: MiscService, @InjectQueue(QUEUE_SKILL) private skillQueue: Queue, @InjectQueue(QUEUE_SKILL_TIMEOUT_CHECK) private timeoutCheckQueue: Queue, @@ -752,6 +754,10 @@ export class SkillService { const { input, result } = data; const { resultId, version, actionMeta, tier } = result; + if (input.images?.length > 0) { + input.images = await this.misc.generateImageUrls(input.images); + } + await this.requestUsageQueue.add('syncRequestUsage', { uid: user.uid, tier, diff --git a/packages/ai-workspace-common/package.json b/packages/ai-workspace-common/package.json index 889285c31..b331b5b3c 100644 --- a/packages/ai-workspace-common/package.json +++ b/packages/ai-workspace-common/package.json @@ -9,7 +9,7 @@ }, "scripts": { "preview": "vite preview", - "codegen": "openapi-rq -i ../openapi-schema/schema.yml -o src --noSchemas && biome check ./src/queries --apply" + "codegen": "openapi-rq -i ../openapi-schema/schema.yml -o src --noSchemas && biome check ./src --write" }, "dependencies": { "@ant-design/icons": "^5.5.2", diff --git a/packages/ai-workspace-common/src/queries/common.ts b/packages/ai-workspace-common/src/queries/common.ts index 63b2f02b0..89f4e3e97 100644 --- a/packages/ai-workspace-common/src/queries/common.ts +++ b/packages/ai-workspace-common/src/queries/common.ts @@ -39,7 +39,6 @@ import { getShareContent, getSubscriptionPlans, getSubscriptionUsage, - invokeAction, invokeSkill, listActions, listCanvases, @@ -61,7 +60,6 @@ import { scrape, search, serveStatic, - streamInvokeAction, streamInvokeSkill, unpinSkillInstance, updateCanvas, @@ -480,18 +478,6 @@ export const UseDeleteLabelInstanceKeyFn = (mutationKey?: Array) => [ useDeleteLabelInstanceKey, ...(mutationKey ?? []), ]; -export type InvokeActionMutationResult = Awaited>; -export const useInvokeActionKey = 'InvokeAction'; -export const UseInvokeActionKeyFn = (mutationKey?: Array) => [ - useInvokeActionKey, - ...(mutationKey ?? []), -]; -export type StreamInvokeActionMutationResult = Awaited>; -export const useStreamInvokeActionKey = 'StreamInvokeAction'; -export const UseStreamInvokeActionKeyFn = (mutationKey?: Array) => [ - useStreamInvokeActionKey, - ...(mutationKey ?? []), -]; export type InvokeSkillMutationResult = Awaited>; export const useInvokeSkillKey = 'InvokeSkill'; export const UseInvokeSkillKeyFn = (mutationKey?: Array) => [ diff --git a/packages/ai-workspace-common/src/queries/queries.ts b/packages/ai-workspace-common/src/queries/queries.ts index 6d674ffa5..4ebf69f23 100644 --- a/packages/ai-workspace-common/src/queries/queries.ts +++ b/packages/ai-workspace-common/src/queries/queries.ts @@ -39,7 +39,6 @@ import { getShareContent, getSubscriptionPlans, getSubscriptionUsage, - invokeAction, invokeSkill, listActions, listCanvases, @@ -61,7 +60,6 @@ import { scrape, search, serveStatic, - streamInvokeAction, streamInvokeSkill, unpinSkillInstance, updateCanvas, @@ -141,8 +139,6 @@ import { GetShareContentError, GetSubscriptionPlansError, GetSubscriptionUsageError, - InvokeActionData, - InvokeActionError, InvokeSkillData, InvokeSkillError, ListActionsError, @@ -179,8 +175,6 @@ import { SearchData, SearchError, ServeStaticError, - StreamInvokeActionData, - StreamInvokeActionError, StreamInvokeSkillData, StreamInvokeSkillError, UnpinSkillInstanceData, @@ -1034,40 +1028,6 @@ export const useDeleteLabelInstance = < mutationFn: (clientOptions) => deleteLabelInstance(clientOptions) as unknown as Promise, ...options, }); -export const useInvokeAction = < - TData = Common.InvokeActionMutationResult, - TError = InvokeActionError, - TQueryKey extends Array = unknown[], - TContext = unknown, ->( - mutationKey?: TQueryKey, - options?: Omit< - UseMutationOptions, TContext>, - 'mutationKey' | 'mutationFn' - >, -) => - useMutation, TContext>({ - mutationKey: Common.UseInvokeActionKeyFn(mutationKey), - mutationFn: (clientOptions) => invokeAction(clientOptions) as unknown as Promise, - ...options, - }); -export const useStreamInvokeAction = < - TData = Common.StreamInvokeActionMutationResult, - TError = StreamInvokeActionError, - TQueryKey extends Array = unknown[], - TContext = unknown, ->( - mutationKey?: TQueryKey, - options?: Omit< - UseMutationOptions, TContext>, - 'mutationKey' | 'mutationFn' - >, -) => - useMutation, TContext>({ - mutationKey: Common.UseStreamInvokeActionKeyFn(mutationKey), - mutationFn: (clientOptions) => streamInvokeAction(clientOptions) as unknown as Promise, - ...options, - }); export const useInvokeSkill = < TData = Common.InvokeSkillMutationResult, TError = InvokeSkillError, diff --git a/packages/ai-workspace-common/src/requests/services.gen.ts b/packages/ai-workspace-common/src/requests/services.gen.ts index 29a629b57..f9995b2ca 100644 --- a/packages/ai-workspace-common/src/requests/services.gen.ts +++ b/packages/ai-workspace-common/src/requests/services.gen.ts @@ -125,12 +125,6 @@ import type { DeleteLabelInstanceResponse, ListActionsError, ListActionsResponse, - InvokeActionData, - InvokeActionError, - InvokeActionResponse2, - StreamInvokeActionData, - StreamInvokeActionError, - StreamInvokeActionResponse, GetActionResultData, GetActionResultError, GetActionResultResponse2, @@ -848,36 +842,6 @@ export const listActions = ( }); }; -/** - * Invoke action - * Invoke an action asynchronously - */ -export const invokeAction = ( - options: Options, -) => { - return (options?.client ?? client).post({ - ...options, - url: '/action/invoke', - }); -}; - -/** - * Stream invoke action - * Invoke an action and return SSE stream - */ -export const streamInvokeAction = ( - options: Options, -) => { - return (options?.client ?? client).post< - StreamInvokeActionResponse, - StreamInvokeActionError, - ThrowOnError - >({ - ...options, - url: '/action/streamInvoke', - }); -}; - /** * Get action result * Get action result by result ID diff --git a/packages/ai-workspace-common/src/requests/types.gen.ts b/packages/ai-workspace-common/src/requests/types.gen.ts index d6fddc683..ee46b6254 100644 --- a/packages/ai-workspace-common/src/requests/types.gen.ts +++ b/packages/ai-workspace-common/src/requests/types.gen.ts @@ -1985,6 +1985,10 @@ export type SkillInput = { * User query */ query?: string; + /** + * Image list (storage keys) + */ + images?: Array; }; /** @@ -2179,56 +2183,6 @@ export type ActionContextItem = { }; }; -export type InvokeActionRequest = { - /** - * Action type - */ - actionType?: ActionType; - /** - * Action name - */ - actionName?: string; - /** - * Action input - */ - input?: SkillInput; - /** - * Action invocation context - */ - context?: Array; - /** - * Action config - */ - config?: ActionConfig; - /** - * Canvas ID - */ - canvasId?: string; - /** - * Selected output locale - */ - locale?: string; - /** - * Selected model - */ - modelName?: string; - /** - * Skill job ID (if not provided, a new job will be created) - */ - jobId?: string; - /** - * Trigger ID (typically you don't need to provide this) - */ - triggerId?: string; -}; - -export type InvokeActionResponse = BaseResponse & { - /** - * Skill job ID - */ - jobId?: string; -}; - export type InvokeSkillRequest = { /** * Skill input @@ -2758,7 +2712,11 @@ export type UploadResponse = BaseResponse & { /** * File URL */ - url?: string; + url: string; + /** + * Storage key + */ + storageKey: string; }; }; @@ -2840,7 +2798,8 @@ export type CanvasNodeType = | 'skillResponse' | 'toolResponse' | 'memo' - | 'group'; + | 'group' + | 'image'; export type CanvasNodeData = { /** @@ -3302,28 +3261,6 @@ export type ListActionsResponse = ListActionResponse; export type ListActionsError = unknown; -export type InvokeActionData = { - /** - * Action invocation request - */ - body: InvokeActionRequest; -}; - -export type InvokeActionResponse2 = InvokeActionResponse; - -export type InvokeActionError = unknown; - -export type StreamInvokeActionData = { - /** - * Skill invocation request - */ - body: InvokeActionRequest; -}; - -export type StreamInvokeActionResponse = string; - -export type StreamInvokeActionError = unknown; - export type GetActionResultData = { query: { /** diff --git a/packages/openapi-schema/openapi-ts.config.ts b/packages/openapi-schema/openapi-ts.config.ts index 925b1529f..c63068809 100644 --- a/packages/openapi-schema/openapi-ts.config.ts +++ b/packages/openapi-schema/openapi-ts.config.ts @@ -4,7 +4,7 @@ export default defineConfig({ client: '@hey-api/client-fetch', input: './schema.yml', output: { - format: 'prettier', + format: 'biome', path: 'src/', }, }); diff --git a/packages/openapi-schema/schema.yml b/packages/openapi-schema/schema.yml index b90ee0f44..5329436c1 100644 --- a/packages/openapi-schema/schema.yml +++ b/packages/openapi-schema/schema.yml @@ -940,48 +940,6 @@ paths: application/json: schema: $ref: '#/components/schemas/ListActionResponse' - /action/invoke: - post: - tags: - - action - summary: Invoke action - description: Invoke an action asynchronously - operationId: invokeAction - requestBody: - description: Action invocation request - content: - application/json: - schema: - $ref: '#/components/schemas/InvokeActionRequest' - required: true - responses: - '200': - description: successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/InvokeActionResponse' - /action/streamInvoke: - post: - tags: - - action - summary: Stream invoke action - description: Invoke an action and return SSE stream - operationId: streamInvokeAction - requestBody: - description: Skill invocation request - content: - application/json: - schema: - $ref: '#/components/schemas/InvokeActionRequest' - required: true - responses: - '200': - description: successful operation - content: - text/event-stream: - schema: - type: string /action/result: get: tags: @@ -3600,6 +3558,11 @@ components: query: type: string description: User query + images: + type: array + description: Image list (storage keys) + items: + type: string SkillContextResourceItem: type: object description: Skill context resource item @@ -3784,49 +3747,6 @@ components: metadata: type: object description: Context metadata - InvokeActionRequest: - type: object - properties: - actionType: - description: Action type - $ref: '#/components/schemas/ActionType' - actionName: - type: string - description: Action name - input: - description: Action input - $ref: '#/components/schemas/SkillInput' - context: - type: array - description: Action invocation context - items: - $ref: '#/components/schemas/ActionContextItem' - config: - description: Action config - $ref: '#/components/schemas/ActionConfig' - canvasId: - description: Canvas ID - type: string - locale: - type: string - description: Selected output locale - modelName: - type: string - description: Selected model - jobId: - description: Skill job ID (if not provided, a new job will be created) - type: string - triggerId: - description: Trigger ID (typically you don't need to provide this) - type: string - InvokeActionResponse: - allOf: - - $ref: '#/components/schemas/BaseResponse' - - type: object - properties: - jobId: - type: string - description: Skill job ID InvokeSkillRequest: type: object properties: @@ -4374,10 +4294,16 @@ components: data: type: object description: File upload result + required: + - url + - storageKey properties: url: type: string description: File URL + storageKey: + type: string + description: Storage key ModelCapabilities: type: object properties: diff --git a/packages/openapi-schema/src/schemas.gen.ts b/packages/openapi-schema/src/schemas.gen.ts index f18100991..bddcc00f8 100644 --- a/packages/openapi-schema/src/schemas.gen.ts +++ b/packages/openapi-schema/src/schemas.gen.ts @@ -2837,6 +2837,13 @@ export const SkillInputSchema = { type: 'string', description: 'User query', }, + images: { + type: 'array', + description: 'Image list (storage keys)', + items: { + type: 'string', + }, + }, }, } as const; @@ -3088,72 +3095,6 @@ export const ActionContextItemSchema = { }, } as const; -export const InvokeActionRequestSchema = { - type: 'object', - properties: { - actionType: { - description: 'Action type', - $ref: '#/components/schemas/ActionType', - }, - actionName: { - type: 'string', - description: 'Action name', - }, - input: { - description: 'Action input', - $ref: '#/components/schemas/SkillInput', - }, - context: { - type: 'array', - description: 'Action invocation context', - items: { - $ref: '#/components/schemas/ActionContextItem', - }, - }, - config: { - description: 'Action config', - $ref: '#/components/schemas/ActionConfig', - }, - canvasId: { - description: 'Canvas ID', - type: 'string', - }, - locale: { - type: 'string', - description: 'Selected output locale', - }, - modelName: { - type: 'string', - description: 'Selected model', - }, - jobId: { - description: 'Skill job ID (if not provided, a new job will be created)', - type: 'string', - }, - triggerId: { - description: "Trigger ID (typically you don't need to provide this)", - type: 'string', - }, - }, -} as const; - -export const InvokeActionResponseSchema = { - allOf: [ - { - $ref: '#/components/schemas/BaseResponse', - }, - { - type: 'object', - properties: { - jobId: { - type: 'string', - description: 'Skill job ID', - }, - }, - }, - ], -} as const; - export const InvokeSkillRequestSchema = { type: 'object', properties: { @@ -3978,11 +3919,16 @@ export const UploadResponseSchema = { data: { type: 'object', description: 'File upload result', + required: ['url', 'storageKey'], properties: { url: { type: 'string', description: 'File URL', }, + storageKey: { + type: 'string', + description: 'Storage key', + }, }, }, }, @@ -4101,7 +4047,17 @@ export const InMemorySearchResponseSchema = { export const CanvasNodeTypeSchema = { type: 'string', - enum: ['document', 'resource', 'skill', 'tool', 'skillResponse', 'toolResponse', 'memo', 'group'], + enum: [ + 'document', + 'resource', + 'skill', + 'tool', + 'skillResponse', + 'toolResponse', + 'memo', + 'group', + 'image', + ], } as const; export const CanvasNodeDataSchema = { diff --git a/packages/openapi-schema/src/services.gen.ts b/packages/openapi-schema/src/services.gen.ts index 29a629b57..f9995b2ca 100644 --- a/packages/openapi-schema/src/services.gen.ts +++ b/packages/openapi-schema/src/services.gen.ts @@ -125,12 +125,6 @@ import type { DeleteLabelInstanceResponse, ListActionsError, ListActionsResponse, - InvokeActionData, - InvokeActionError, - InvokeActionResponse2, - StreamInvokeActionData, - StreamInvokeActionError, - StreamInvokeActionResponse, GetActionResultData, GetActionResultError, GetActionResultResponse2, @@ -848,36 +842,6 @@ export const listActions = ( }); }; -/** - * Invoke action - * Invoke an action asynchronously - */ -export const invokeAction = ( - options: Options, -) => { - return (options?.client ?? client).post({ - ...options, - url: '/action/invoke', - }); -}; - -/** - * Stream invoke action - * Invoke an action and return SSE stream - */ -export const streamInvokeAction = ( - options: Options, -) => { - return (options?.client ?? client).post< - StreamInvokeActionResponse, - StreamInvokeActionError, - ThrowOnError - >({ - ...options, - url: '/action/streamInvoke', - }); -}; - /** * Get action result * Get action result by result ID diff --git a/packages/openapi-schema/src/types.gen.ts b/packages/openapi-schema/src/types.gen.ts index d6fddc683..ee46b6254 100644 --- a/packages/openapi-schema/src/types.gen.ts +++ b/packages/openapi-schema/src/types.gen.ts @@ -1985,6 +1985,10 @@ export type SkillInput = { * User query */ query?: string; + /** + * Image list (storage keys) + */ + images?: Array; }; /** @@ -2179,56 +2183,6 @@ export type ActionContextItem = { }; }; -export type InvokeActionRequest = { - /** - * Action type - */ - actionType?: ActionType; - /** - * Action name - */ - actionName?: string; - /** - * Action input - */ - input?: SkillInput; - /** - * Action invocation context - */ - context?: Array; - /** - * Action config - */ - config?: ActionConfig; - /** - * Canvas ID - */ - canvasId?: string; - /** - * Selected output locale - */ - locale?: string; - /** - * Selected model - */ - modelName?: string; - /** - * Skill job ID (if not provided, a new job will be created) - */ - jobId?: string; - /** - * Trigger ID (typically you don't need to provide this) - */ - triggerId?: string; -}; - -export type InvokeActionResponse = BaseResponse & { - /** - * Skill job ID - */ - jobId?: string; -}; - export type InvokeSkillRequest = { /** * Skill input @@ -2758,7 +2712,11 @@ export type UploadResponse = BaseResponse & { /** * File URL */ - url?: string; + url: string; + /** + * Storage key + */ + storageKey: string; }; }; @@ -2840,7 +2798,8 @@ export type CanvasNodeType = | 'skillResponse' | 'toolResponse' | 'memo' - | 'group'; + | 'group' + | 'image'; export type CanvasNodeData = { /** @@ -3302,28 +3261,6 @@ export type ListActionsResponse = ListActionResponse; export type ListActionsError = unknown; -export type InvokeActionData = { - /** - * Action invocation request - */ - body: InvokeActionRequest; -}; - -export type InvokeActionResponse2 = InvokeActionResponse; - -export type InvokeActionError = unknown; - -export type StreamInvokeActionData = { - /** - * Skill invocation request - */ - body: InvokeActionRequest; -}; - -export type StreamInvokeActionResponse = string; - -export type StreamInvokeActionError = unknown; - export type GetActionResultData = { query: { /** diff --git a/packages/skill-template/src/base.ts b/packages/skill-template/src/base.ts index 807ebbfb4..9c460f576 100644 --- a/packages/skill-template/src/base.ts +++ b/packages/skill-template/src/base.ts @@ -135,6 +135,10 @@ export const baseStateGraphArgs = { reducer: (left: string, right: string) => (right ? right : left || ''), default: () => '', }, + images: { + reducer: (x: string[], y: string[]) => x.concat(y), + default: () => [], + }, locale: { reducer: (left?: string, right?: string) => (right ? right : left || 'en'), default: () => 'en', diff --git a/packages/skill-template/src/scheduler/utils/message.ts b/packages/skill-template/src/scheduler/utils/message.ts index d0a0fb405..1964866d0 100644 --- a/packages/skill-template/src/scheduler/utils/message.ts +++ b/packages/skill-template/src/scheduler/utils/message.ts @@ -21,6 +21,7 @@ export const buildFinalRequestMessages = ({ messages, needPrepareContext, context, + images, originalQuery, rewrittenQuery, }: { @@ -30,6 +31,7 @@ export const buildFinalRequestMessages = ({ messages: BaseMessage[]; needPrepareContext: boolean; context: string; + images: string[]; originalQuery: string; rewrittenQuery: string; }) => { @@ -44,7 +46,12 @@ export const buildFinalRequestMessages = ({ ...chatHistory, ...messages, ...contextMessages, - new HumanMessage(userPrompt), + new HumanMessage({ + content: [ + { type: 'text', text: userPrompt }, + ...(images?.map((image) => ({ type: 'image_url', image_url: { url: image } })) || []), + ], + }), ]; return requestMessages; diff --git a/packages/skill-template/src/skills/common-qna.ts b/packages/skill-template/src/skills/common-qna.ts index 04d2f7aed..f9a3efd93 100644 --- a/packages/skill-template/src/skills/common-qna.ts +++ b/packages/skill-template/src/skills/common-qna.ts @@ -56,7 +56,7 @@ export class CommonQnA extends BaseSkill { config: SkillRunnableConfig, module: SkillPromptModule, ) => { - const { messages = [] } = state; + const { messages = [], images = [] } = state; const { locale = 'en', modelInfo } = config.configurable; // Use shared query processor @@ -112,6 +112,7 @@ export class CommonQnA extends BaseSkill { messages, needPrepareContext: needPrepareContext && isModelContextLenSupport, context, + images, originalQuery: query, rewrittenQuery: optimizedQuery, }); diff --git a/packages/skill-template/src/skills/edit-doc.ts b/packages/skill-template/src/skills/edit-doc.ts index 725dbf2e7..99c86168d 100644 --- a/packages/skill-template/src/skills/edit-doc.ts +++ b/packages/skill-template/src/skills/edit-doc.ts @@ -68,7 +68,7 @@ export class EditDoc extends BaseSkill { config: SkillRunnableConfig, module: SkillPromptModule, ) => { - const { messages = [], query: originalQuery } = state; + const { messages = [], query: originalQuery, images = [] } = state; const { locale = 'en', chatHistory = [], @@ -115,13 +115,12 @@ export class EditDoc extends BaseSkill { `maxTokens: ${maxTokens}, queryTokens: ${queryTokens}, chatHistoryTokens: ${chatHistoryTokens}, remainingTokens: ${remainingTokens}`, ); - // 新增：定义长查询的阈值（可以根据实际需求调整） - const LONG_QUERY_TOKENS_THRESHOLD = 100; // 约等于50-75个英文单词或25-35个中文字 + const LONG_QUERY_TOKENS_THRESHOLD = 100; // About 50-75 English words or 25-35 Chinese characters - // 优化 needRewriteQuery 判断逻辑 + // Optimize needRewriteQuery judgment logic const needRewriteQuery = - queryTokens < LONG_QUERY_TOKENS_THRESHOLD && // 只有短查询才需要重写 - (hasContext || chatHistoryTokens > 0); // 保持原有的上下文相关判断 + queryTokens < LONG_QUERY_TOKENS_THRESHOLD && // Only rewrite short queries + (hasContext || chatHistoryTokens > 0); // Keep original context-related judgment const needPrepareContext = hasContext && remainingTokens > 0; this.engine.logger.log( @@ -177,6 +176,7 @@ export class EditDoc extends BaseSkill { messages, needPrepareContext: needPrepareContext && isModelContextLenSupport, context, + images, originalQuery: query, rewrittenQuery: optimizedQuery, }); @@ -186,12 +186,6 @@ export class EditDoc extends BaseSkill { return { requestMessages }; }; - // TODO: 将实际的 document 的内容发送给模型，拼接为 prompt 处理 - /** - * Update canvas：更新的形态 - * 1. 口头模糊指明（可能涉及处理多个）：直接口头指明模糊更新的内容（需要模型扫描并给出待操作的模块和对应的 startIndex 和 endIndex），则只需要优化这些内容，其他保持原样，并且发送给前端流式写入 - * 2. 前端明确选中（目前只支持一个）：明确具备选中的 startIndex 和 endIndex（使用的是 tiptap editor），则只需要优化这块内容，其他保持原样，并且发送给前端流式写入 - */ callEditDoc = async ( state: GraphState, config: SkillRunnableConfig, diff --git a/packages/skill-template/src/skills/generate-doc.ts b/packages/skill-template/src/skills/generate-doc.ts index 3f06a305a..66870683c 100644 --- a/packages/skill-template/src/skills/generate-doc.ts +++ b/packages/skill-template/src/skills/generate-doc.ts @@ -60,7 +60,7 @@ export class GenerateDoc extends BaseSkill { config: SkillRunnableConfig, module: SkillPromptModule, ) => { - const { messages = [] } = state; + const { messages = [], images = [] } = state; const { locale = 'en', modelInfo } = config.configurable; const { tplConfig } = config?.configurable || {}; @@ -119,6 +119,7 @@ export class GenerateDoc extends BaseSkill { messages, needPrepareContext: needPrepareContext && isModelContextLenSupport, context, + images, originalQuery: query, rewrittenQuery: optimizedQuery, }); diff --git a/packages/skill-template/src/skills/library-search.ts b/packages/skill-template/src/skills/library-search.ts index c3167d262..0bf3426e6 100644 --- a/packages/skill-template/src/skills/library-search.ts +++ b/packages/skill-template/src/skills/library-search.ts @@ -43,7 +43,7 @@ export class LibrarySearch extends BaseSkill { state: GraphState, config: SkillRunnableConfig, ): Promise> => { - const { messages = [] } = state; + const { messages = [], images = [] } = state; const { locale = 'en', currentSkill } = config.configurable; // Set current step @@ -111,6 +111,7 @@ export class LibrarySearch extends BaseSkill { messages, needPrepareContext: true, context: contextStr, + images, originalQuery: query, rewrittenQuery: optimizedQuery, }); diff --git a/packages/skill-template/src/skills/rewrite-doc.ts b/packages/skill-template/src/skills/rewrite-doc.ts index 520140306..dfa1fbf33 100644 --- a/packages/skill-template/src/skills/rewrite-doc.ts +++ b/packages/skill-template/src/skills/rewrite-doc.ts @@ -58,7 +58,7 @@ export class RewriteDoc extends BaseSkill { config: SkillRunnableConfig, module: SkillPromptModule, ) => { - const { messages = [], query: originalQuery } = state; + const { messages = [], query: originalQuery, images = [] } = state; const { locale = 'en', chatHistory = [], @@ -105,13 +105,12 @@ export class RewriteDoc extends BaseSkill { `maxTokens: ${maxTokens}, queryTokens: ${queryTokens}, chatHistoryTokens: ${chatHistoryTokens}, remainingTokens: ${remainingTokens}`, ); - // 新增：定义长查询的阈值（可以根据实际需求调整） - const LONG_QUERY_TOKENS_THRESHOLD = 100; // 约等于50-75个英文单词或25-35个中文字 + const LONG_QUERY_TOKENS_THRESHOLD = 100; // About 50-75 English words or 25-35 Chinese characters - // 优化 needRewriteQuery 判断逻辑 + // Optimize needRewriteQuery judgment logic const needRewriteQuery = - queryTokens < LONG_QUERY_TOKENS_THRESHOLD && // 只有短查询才需要重写 - (hasContext || chatHistoryTokens > 0); // 保持原有的上下文相关判断 + queryTokens < LONG_QUERY_TOKENS_THRESHOLD && // Only rewrite short queries + (hasContext || chatHistoryTokens > 0); // Keep original context-related judgment const needPrepareContext = (hasContext && remainingTokens > 0) || enableWebSearch || enableKnowledgeBaseSearch; @@ -165,6 +164,7 @@ export class RewriteDoc extends BaseSkill { messages, needPrepareContext, context, + images, originalQuery: query, rewrittenQuery: optimizedQuery, }); diff --git a/packages/skill-template/src/skills/web-search.ts b/packages/skill-template/src/skills/web-search.ts index 72e05886d..0891971ed 100644 --- a/packages/skill-template/src/skills/web-search.ts +++ b/packages/skill-template/src/skills/web-search.ts @@ -58,7 +58,7 @@ export class WebSearch extends BaseSkill { state: GraphState, config: SkillRunnableConfig, ): Promise> => { - const { messages = [] } = state; + const { messages = [], images = [] } = state; const { locale = 'en', currentSkill } = config.configurable; // Set current step @@ -122,6 +122,7 @@ export class WebSearch extends BaseSkill { messages, needPrepareContext: true, context: contextStr, + images, originalQuery: query, rewrittenQuery: optimizedQuery, }); From 0f7705cce196a91b76cc419014e0c961652aabf5 Mon Sep 17 00:00:00 2001 From: mrcfps Date: Fri, 7 Feb 2025 10:46:05 +0800 Subject: [PATCH 04/83] fix(api): type-only service import --- .vscode/settings.json | 1 - apps/api/src/skill/skill.service.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index f7a0ad027..93daa6ec0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,7 +6,6 @@ "editor.formatOnSave": true, "editor.defaultFormatter": "biomejs.biome", "typescript.preferences.importModuleSpecifier": "non-relative", - "typescript.preferences.preferTypeOnlyAutoImports": true, "tailwindCSS.includeLanguages": { "typescript": "tsx", "javascript": "jsx", diff --git a/apps/api/src/skill/skill.service.ts b/apps/api/src/skill/skill.service.ts index 9a61647c1..41ffa1f22 100644 --- a/apps/api/src/skill/skill.service.ts +++ b/apps/api/src/skill/skill.service.ts @@ -88,7 +88,7 @@ import { ResultAggregator } from '@/utils/result'; import { CollabContext } from '@/collab/collab.dto'; import { DirectConnection } from '@hocuspocus/server'; import { modelInfoPO2DTO } from '@/misc/misc.dto'; -import type { MiscService } from '@/misc/misc.service'; +import { MiscService } from '@/misc/misc.service'; export function createLangchainMessage(result: ActionResult, steps: ActionStep[]): BaseMessage[] { const query = result.title; From e5d1417b70c58ccb4aa80399a89f7a0aaeb2b55f Mon Sep 17 00:00:00 2001 From: CH Date: Fri, 7 Feb 2025 12:00:31 +0800 Subject: [PATCH 05/83] feat(canvas): Improve image node handling and copy functionality - Add server-side image upload for drag and drop - Implement image copying to clipboard with cross-origin support - Update node action menu to handle image-specific actions - Modify image node styling for better scrolling --- .../src/components/canvas/index.tsx | 35 ++++++----- .../canvas/node-action-menu/index.tsx | 60 +++++++++++++++---- .../src/components/canvas/nodes/image.tsx | 2 +- 3 files changed, 71 insertions(+), 26 deletions(-) diff --git a/packages/ai-workspace-common/src/components/canvas/index.tsx b/packages/ai-workspace-common/src/components/canvas/index.tsx index c71d9bdbd..a420fc56a 100644 --- a/packages/ai-workspace-common/src/components/canvas/index.tsx +++ b/packages/ai-workspace-common/src/components/canvas/index.tsx @@ -1,7 +1,7 @@ import { useCallback, useMemo, useEffect, useState, useRef, memo } from 'react'; import { useTranslation } from 'react-i18next'; import { ReactFlow, Background, MiniMap, ReactFlowProvider, useReactFlow } from '@xyflow/react'; -import { Button } from 'antd'; +import { Button, message } from 'antd'; import { nodeTypes, CanvasNode } from './nodes'; import { LaunchPad } from './launchpad'; import { CanvasToolbar } from './canvas-toolbar'; @@ -47,6 +47,7 @@ import { SelectionContextMenu } from '@refly-packages/ai-workspace-common/compon import { useUserStore } from '@refly-packages/ai-workspace-common/stores/user'; import { useUpdateSettings } from '@refly-packages/ai-workspace-common/queries'; import { IconCreateDocument } from '@refly-packages/ai-workspace-common/components/common/icon'; +import getClient from '@refly-packages/ai-workspace-common/requests/proxiedRequest'; const selectionStyles = ` .react-flow__selection { @@ -488,6 +489,17 @@ const Flow = memo(({ canvasId }: { canvasId: string }) => { [reactFlowInstance], ); + const uploadImage = async (image: File) => { + const response = await getClient().upload({ + body: { + file: image, + entityId: canvasId, + entityType: 'canvas', + }, + }); + return response.data; + }; + // Add drag and drop handlers const handleDragOver = useCallback((event: React.DragEvent) => { event.preventDefault(); @@ -495,38 +507,33 @@ const Flow = memo(({ canvasId }: { canvasId: string }) => { }, []); const handleDrop = useCallback( - (event: React.DragEvent) => { + async (event: React.DragEvent) => { event.preventDefault(); - const files = Array.from(event.dataTransfer.files); const imageFile = files.find((file) => file.type.startsWith('image/')); if (imageFile) { - const reader = new FileReader(); - reader.onload = (e) => { - const imageUrl = e.target?.result as string; - - // Get drop position in flow coordinates + const { data, success } = await uploadImage(imageFile); + if (success) { const flowPosition = reactFlowInstance.screenToFlowPosition({ x: event.clientX, y: event.clientY, }); - - // Add image node addNode({ type: 'image', data: { title: imageFile.name, entityId: genImageID(), metadata: { - imageUrl, - sizeMode: 'adaptive', + imageUrl: data.url, + storageKey: data.storageKey, }, }, position: flowPosition, }); - }; - reader.readAsDataURL(imageFile); + } else { + message.error(t('common.putErr')); + } } }, [addNode, reactFlowInstance], diff --git a/packages/ai-workspace-common/src/components/canvas/node-action-menu/index.tsx b/packages/ai-workspace-common/src/components/canvas/node-action-menu/index.tsx index 7daa28399..8caf6d83f 100644 --- a/packages/ai-workspace-common/src/components/canvas/node-action-menu/index.tsx +++ b/packages/ai-workspace-common/src/components/canvas/node-action-menu/index.tsx @@ -222,12 +222,52 @@ export const NodeActionMenu: FC = ({ }, [nodeId, nodeType, layoutNodeCluster, onClose]); const handleCopy = useCallback(() => { - const content = nodeData?.contentPreview; - - copyToClipboard(content || ''); - message.success(t('copilot.message.copySuccess')); + if (nodeType === 'image' && nodeData?.metadata?.imageUrl) { + const copyImage = async () => { + try { + const img = new Image(); + img.crossOrigin = 'anonymous'; + + await new Promise((resolve, reject) => { + img.onload = resolve; + img.onerror = reject; + img.src = nodeData.metadata.imageUrl as string; + }); + + const canvas = document.createElement('canvas'); + canvas.width = img.width; + canvas.height = img.height; + const ctx = canvas.getContext('2d'); + ctx?.drawImage(img, 0, 0); + + canvas.toBlob(async (blob) => { + if (blob) { + try { + await navigator.clipboard.write([ + new ClipboardItem({ + 'image/png': blob, + }), + ]); + message.success(t('copilot.message.copySuccess')); + } catch (error) { + console.error('Failed to copy image:', error); + message.error(t('copilot.message.copyFailed')); + } + } + }, 'image/png'); + } catch (error) { + console.error('Failed to load image:', error); + message.error(t('copilot.message.copyFailed')); + } + }; + copyImage(); + } else { + const content = nodeData?.contentPreview; + copyToClipboard(content || ''); + message.success(t('copilot.message.copySuccess')); + } onClose?.(); - }, [nodeData?.contentPreview, onClose, t]); + }, [nodeData?.contentPreview, nodeData?.metadata?.imageUrl, onClose, t, nodeType]); const handleEditQuery = useCallback(() => { addNodePreview(canvasId, node); @@ -394,8 +434,8 @@ export const NodeActionMenu: FC = ({ 'https://static.refly.ai/onboarding/nodeAction/nodeActionMenu-openPreview.webm', }, }, - { key: 'divider-1', type: 'divider' } as MenuItem, - { + nodeType !== 'image' && ({ key: 'divider-1', type: 'divider' } as MenuItem), + nodeType !== 'image' && { key: 'toggleSizeMode', icon: localSizeMode === 'compact' ? IconExpand : IconShrink, label: @@ -657,11 +697,9 @@ export const NodeActionMenu: FC = ({ return [ ...(nodeType !== 'skill' ? commonItems : []), - ...(nodeType !== 'memo' && nodeType !== 'skill' && nodeType !== 'group' - ? operationItems - : []), + ...(!['memo', 'skill', 'group'].includes(nodeType) ? operationItems : []), ...(nodeTypeItems[nodeType] || []), - ...(nodeType !== 'memo' && nodeType !== 'skill' ? clusterItems : []), + ...(!['memo', 'skill', 'image'].includes(nodeType) ? clusterItems : []), { key: 'divider-3', type: 'divider' } as MenuItem, ...footerItems, ].filter(Boolean); diff --git a/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx b/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx index 927823d1c..da37ff32e 100644 --- a/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx +++ b/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx @@ -213,7 +213,7 @@ export const ImageNode = memo(
-
+
{data.title Date: Fri, 7 Feb 2025 14:58:59 +0800 Subject: [PATCH 06/83] feat(canvas): Refactor image node types and preview rendering - Update image node type definitions in shared types - Add image node to context preview rendering - Simplify image node component imports and props - Adjust image preview modal rendering logic --- .../context-manager/context-preview.tsx | 4 +++ .../src/components/canvas/nodes/image.tsx | 31 ++++++------------- .../src/components/canvas/nodes/index.ts | 2 +- .../components/canvas/nodes/shared/types.ts | 4 +++ 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/packages/ai-workspace-common/src/components/canvas/launchpad/context-manager/context-preview.tsx b/packages/ai-workspace-common/src/components/canvas/launchpad/context-manager/context-preview.tsx index f39b53f1e..2fe1ddd7c 100644 --- a/packages/ai-workspace-common/src/components/canvas/launchpad/context-manager/context-preview.tsx +++ b/packages/ai-workspace-common/src/components/canvas/launchpad/context-manager/context-preview.tsx @@ -5,11 +5,13 @@ import { ResourceNodeProps, SkillResponseNode, SkillResponseNodeProps, + ImageNodeProps, } from '@refly-packages/ai-workspace-common/components/canvas/nodes'; import { DocumentNode, ResourceNode, MemoNode, + ImageNode, } from '@refly-packages/ai-workspace-common/components/canvas/nodes'; import { useCanvasData } from '@refly-packages/ai-workspace-common/hooks/canvas/use-canvas-data'; import { IContextItem } from '@refly-packages/ai-workspace-common/stores/context-panel'; @@ -50,6 +52,8 @@ export const ContextPreview = memo( case 'documentSelection': case 'skillResponseSelection': return ; + case 'image': + return ; default: return null; } diff --git a/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx b/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx index da37ff32e..69d3a485d 100644 --- a/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx +++ b/packages/ai-workspace-common/src/components/canvas/nodes/image.tsx @@ -1,7 +1,7 @@ -import React, { memo, useState, useCallback, useRef, useMemo, useEffect } from 'react'; -import { NodeProps, useReactFlow, Position } from '@xyflow/react'; +import { memo, useState, useCallback, useRef, useMemo, useEffect } from 'react'; +import { useReactFlow, Position } from '@xyflow/react'; import { Image } from 'antd'; -import { CanvasNode, CommonNodeProps } from './shared/types'; +import { CanvasNode, ImageNodeProps } from './shared/types'; import { ActionButtons } from './shared/action-buttons'; import { useNodeHoverEffect } from '@refly-packages/ai-workspace-common/hooks/canvas/use-node-hover'; import { useNodeSize } from '@refly-packages/ai-workspace-common/hooks/canvas/use-node-size'; @@ -24,17 +24,6 @@ import { useAddToContext } from '@refly-packages/ai-workspace-common/hooks/canva import { useDeleteNode } from '@refly-packages/ai-workspace-common/hooks/canvas/use-delete-node'; import Moveable from 'react-moveable'; -// Define image node metadata type -export interface ImageNodeMeta { - imageUrl: string; - sizeMode?: 'compact' | 'adaptive'; - style?: React.CSSProperties; - originalWidth?: number; -} - -// Define image node props type -export type ImageNodeProps = NodeProps> & CommonNodeProps; - export const ImageNode = memo( ({ id, data, isPreview, selected, hideActions, hideHandles, onNodeClick }: ImageNodeProps) => { const { metadata } = data ?? {}; @@ -53,13 +42,11 @@ export const ImageNode = memo( })); const isOperating = operatingNodeId === id; - const sizeMode = metadata?.sizeMode || 'adaptive'; const node = useMemo(() => getNode(id), [id, getNode]); const { containerStyle, handleResize } = useNodeSize({ id, node, - sizeMode, isOperating, minWidth: 100, maxWidth: 800, @@ -159,9 +146,10 @@ export const ImageNode = memo( useEffect(() => { if (!targetRef.current) return; - - const { offsetWidth, offsetHeight } = targetRef.current; - resizeMoveable(offsetWidth, offsetHeight); + setTimeout(() => { + const { offsetWidth, offsetHeight } = targetRef.current; + resizeMoveable(offsetWidth, offsetHeight); + }, 1); }, [resizeMoveable, targetRef.current?.offsetHeight]); if (!data || !imageUrl) { @@ -221,7 +209,7 @@ export const ImageNode = memo( /> {/* only for preview image */} - {isPreviewModalVisible && ( + {isPreviewModalVisible && !isPreview && (
- {!isPreview && selected && sizeMode === 'adaptive' && ( + {!isPreview && selected && ( )} diff --git a/packages/ai-workspace-common/src/components/canvas/nodes/index.ts b/packages/ai-workspace-common/src/components/canvas/nodes/index.ts index 53d432b4a..c5bb7c326 100644 --- a/packages/ai-workspace-common/src/components/canvas/nodes/index.ts +++ b/packages/ai-workspace-common/src/components/canvas/nodes/index.ts @@ -29,7 +29,7 @@ export * from './tool'; export * from './skill-response'; export * from './memo/memo'; export * from './group'; - +export * from './image'; // Node types mapping export const nodeTypes: NodeTypes = { document: DocumentNode, diff --git a/packages/ai-workspace-common/src/components/canvas/nodes/shared/types.ts b/packages/ai-workspace-common/src/components/canvas/nodes/shared/types.ts index a9481f483..4706e31be 100644 --- a/packages/ai-workspace-common/src/components/canvas/nodes/shared/types.ts +++ b/packages/ai-workspace-common/src/components/canvas/nodes/shared/types.ts @@ -73,6 +73,8 @@ export type ResponseNodeMeta = { export type ImageNodeMeta = { imageType: string; + imageUrl: string; + storageKey: string; }; // Type mapping for node metadata @@ -103,3 +105,5 @@ export type SkillResponseNodeProps = NodeProps< > & CommonNodeProps; export type MemoNodeProps = NodeProps> & CommonNodeProps; +export type ImageNodeProps = NodeProps, 'image'>> & + CommonNodeProps; From cbdfcddb16f4cd11d0ee45e95dfde65e427154af Mon Sep 17 00:00:00 2001 From: CH Date: Fri, 7 Feb 2025 16:00:58 +0800 Subject: [PATCH 07/83] feat(canvas): Centralize image upload functionality with new hook - Create `useUploadImage` hook to standardize image upload process - Add image upload support to chat panel, skill node, and canvas components - Update translations with image upload related messages - Remove direct image upload implementations in favor of centralized hook --- .../src/components/canvas/index.tsx | 38 ++------------ .../canvas/launchpad/chat-actions/index.tsx | 39 ++++++++++---- .../canvas/launchpad/chat-panel.tsx | 17 +++++++ .../skill-response/edit-chat-input.tsx | 17 +++++++ .../src/components/canvas/nodes/skill.tsx | 18 +++++++ .../src/hooks/use-upload-image.ts | 51 +++++++++++++++++++ packages/i18n/src/en-US/ui.ts | 3 ++ packages/i18n/src/zh-Hans/ui.ts | 3 ++ 8 files changed, 141 insertions(+), 45 deletions(-) create mode 100644 packages/ai-workspace-common/src/hooks/use-upload-image.ts diff --git a/packages/ai-workspace-common/src/components/canvas/index.tsx b/packages/ai-workspace-common/src/components/canvas/index.tsx index a420fc56a..0f959e30b 100644 --- a/packages/ai-workspace-common/src/components/canvas/index.tsx +++ b/packages/ai-workspace-common/src/components/canvas/index.tsx @@ -1,7 +1,7 @@ import { useCallback, useMemo, useEffect, useState, useRef, memo } from 'react'; import { useTranslation } from 'react-i18next'; import { ReactFlow, Background, MiniMap, ReactFlowProvider, useReactFlow } from '@xyflow/react'; -import { Button, message } from 'antd'; +import { Button } from 'antd'; import { nodeTypes, CanvasNode } from './nodes'; import { LaunchPad } from './launchpad'; import { CanvasToolbar } from './canvas-toolbar'; @@ -13,7 +13,6 @@ import { useCreateDocument } from '@refly-packages/ai-workspace-common/hooks/can import { useNodeOperations } from '@refly-packages/ai-workspace-common/hooks/canvas/use-node-operations'; import { useNodeSelection } from '@refly-packages/ai-workspace-common/hooks/canvas/use-node-selection'; import { useAddNode } from '@refly-packages/ai-workspace-common/hooks/canvas/use-add-node'; -import { genImageID } from '@refly-packages/utils/id'; import { CanvasProvider, useCanvasContext, @@ -47,7 +46,7 @@ import { SelectionContextMenu } from '@refly-packages/ai-workspace-common/compon import { useUserStore } from '@refly-packages/ai-workspace-common/stores/user'; import { useUpdateSettings } from '@refly-packages/ai-workspace-common/queries'; import { IconCreateDocument } from '@refly-packages/ai-workspace-common/components/common/icon'; -import getClient from '@refly-packages/ai-workspace-common/requests/proxiedRequest'; +import { useUploadImage } from '@refly-packages/ai-workspace-common/hooks/use-upload-image'; const selectionStyles = ` .react-flow__selection { @@ -489,16 +488,7 @@ const Flow = memo(({ canvasId }: { canvasId: string }) => { [reactFlowInstance], ); - const uploadImage = async (image: File) => { - const response = await getClient().upload({ - body: { - file: image, - entityId: canvasId, - entityType: 'canvas', - }, - }); - return response.data; - }; + const { handleUploadImage } = useUploadImage(); // Add drag and drop handlers const handleDragOver = useCallback((event: React.DragEvent) => { @@ -513,27 +503,7 @@ const Flow = memo(({ canvasId }: { canvasId: string }) => { const imageFile = files.find((file) => file.type.startsWith('image/')); if (imageFile) { - const { data, success } = await uploadImage(imageFile); - if (success) { - const flowPosition = reactFlowInstance.screenToFlowPosition({ - x: event.clientX, - y: event.clientY, - }); - addNode({ - type: 'image', - data: { - title: imageFile.name, - entityId: genImageID(), - metadata: { - imageUrl: data.url, - storageKey: data.storageKey, - }, - }, - position: flowPosition, - }); - } else { - message.error(t('common.putErr')); - } + handleUploadImage(imageFile, canvasId, event); } }, [addNode, reactFlowInstance], diff --git a/packages/ai-workspace-common/src/components/canvas/launchpad/chat-actions/index.tsx b/packages/ai-workspace-common/src/components/canvas/launchpad/chat-actions/index.tsx index 225ca0e95..5f3b3dfaa 100644 --- a/packages/ai-workspace-common/src/components/canvas/launchpad/chat-actions/index.tsx +++ b/packages/ai-workspace-common/src/components/canvas/launchpad/chat-actions/index.tsx @@ -1,16 +1,16 @@ -import { Button, Tooltip } from 'antd'; +import { Button, Tooltip, Upload } from 'antd'; import { FormInstance } from '@arco-design/web-react'; -import { useRef, useMemo, useCallback } from 'react'; -import { memo } from 'react'; - +import { memo, useMemo, useRef } from 'react'; +import { IconImage } from '@refly-packages/ai-workspace-common/components/common/icon'; import { IconSend } from '@arco-design/web-react/icon'; import { useTranslation } from 'react-i18next'; import { useUserStoreShallow } from '@refly-packages/ai-workspace-common/stores/user'; - import { getRuntime } from '@refly/utils/env'; import { ModelSelector } from './model-selector'; import { ModelInfo } from '@refly/openapi-schema'; import { cn } from '@refly-packages/utils/index'; +import { useCanvasContext } from '@refly-packages/ai-workspace-common/context/canvas'; +import { useUploadImage } from '@refly-packages/ai-workspace-common/hooks/use-upload-image'; export interface CustomAction { icon: React.ReactNode; @@ -27,20 +27,32 @@ interface ChatActionsProps { handleSendMessage: () => void; handleAbort: () => void; customActions?: CustomAction[]; + onUploadImage?: (file: File) => Promise; } export const ChatActions = memo( (props: ChatActionsProps) => { - const { query, model, setModel, handleSendMessage, customActions, className } = props; + const { query, model, setModel, handleSendMessage, customActions, className, onUploadImage } = + props; const { t } = useTranslation(); + const { canvasId } = useCanvasContext(); + const { handleUploadImage } = useUploadImage(); - const handleSendClick = useCallback(() => { + const handleSendClick = () => { handleSendMessage(); - }, [handleSendMessage]); + }; + + const handleImageUpload = async (file: File) => { + if (onUploadImage) { + await onUploadImage(file); + } else { + await handleUploadImage(file, canvasId); + } + return false; + }; // hooks - const runtime = getRuntime(); - const isWeb = runtime === 'web'; + const isWeb = getRuntime() === 'web'; const userStore = useUserStoreShallow((state) => ({ isLogin: state.isLogin, @@ -68,6 +80,12 @@ export const ChatActions = memo( ))} + + +