Feat/support extension save url title

apps/api/src/knowledge/knowledge.controller.ts

@@ -39,6 +39,7 @@ import { buildSuccessResponse } from '@/utils';
 import { LoginedUser } from '@/utils/decorators/user.decorator';
 import { documentPO2DTO, resourcePO2DTO, referencePO2DTO } from './knowledge.dto';
 import { ParamsError } from '@refly-packages/errors';
+import { safeParseJSON } from '@refly-packages/utils';
 
 @Controller('v1/knowledge')
 export class KnowledgeController {
@@ -101,13 +102,15 @@ export class KnowledgeController {
 
     // Convert file content to string
     const content = file.buffer.toString('utf-8');
+    const data = typeof body.data === 'object' ? body.data : safeParseJSON(body.data);
 
     // Create resource with file content
     const resource = await this.knowledgeService.createResource(
       user,
       {
         ...body,
         content,
+        data,
       },
       {
         checkStorageQuota: true,

apps/api/src/knowledge/knowledge.dto.ts

@@ -2,6 +2,7 @@ import {
   Resource as ResourceModel,
   Document as DocumentModel,
   Reference as ReferenceModel,
+  StaticFile as StaticFileModel,
 } from '@prisma/client';
 import {
   Resource,
@@ -12,9 +13,20 @@ import {
   ReferenceMeta,
   Document,
   EntityType,
+  ResourceMeta,
 } from '@refly-packages/openapi-schema';
 import { pick } from '@/utils';
 
+export interface ResourcePrepareResult {
+  storageKey?: string;
+  staticFile?: StaticFileModel;
+  storageSize?: number;
+  identifier?: string;
+  indexStatus?: IndexStatus;
+  contentPreview?: string;
+  metadata?: ResourceMeta;
+}
+
 export type FinalizeResourceParam = {
   resourceId: string;
   uid: string;

apps/api/src/knowledge/knowledge.service.ts

@@ -21,7 +21,6 @@ import {
   ListResourcesData,
   User,
   GetResourceDetailData,
-  IndexStatus,
   ReindexResourceRequest,
   ResourceType,
   QueryReferencesRequest,
@@ -50,7 +49,11 @@ import {
   genReferenceID,
   genDocumentID,
 } from '@refly-packages/utils';
-import { ExtendedReferenceModel, FinalizeResourceParam } from './knowledge.dto';
+import {
+  ExtendedReferenceModel,
+  FinalizeResourceParam,
+  ResourcePrepareResult,
+} from './knowledge.dto';
 import { pick } from '../utils';
 import { SimpleEventData } from '@/event/event.dto';
 import { SyncStorageUsageJobData } from '@/subscription/subscription.dto';
@@ -63,7 +66,6 @@ import {
   ReferenceNotFoundError,
   ReferenceObjectMissingError,
   DocumentNotFoundError,
-  StaticFileNotFoundError,
 } from '@refly-packages/errors';
 import { DeleteCanvasNodesJobData } from '@/canvas/canvas.dto';
 import { ParserFactory } from '@/knowledge/parsers/factory';
@@ -141,86 +143,117 @@ export class KnowledgeService {
     return { ...resource, content };
   }
 
-  async createResource(
-    user: User,
-    param: UpsertResourceRequest,
-    options?: { checkStorageQuota?: boolean },
-  ) {
-    if (options?.checkStorageQuota) {
-      const usageResult = await this.subscriptionService.checkStorageUsage(user);
-      if (usageResult.available < 1) {
-        throw new StorageQuotaExceeded();
-      }
-    }
-
-    param.resourceId = genResourceID();
+  async prepareResource(user: User, param: UpsertResourceRequest): Promise<ResourcePrepareResult> {
+    const { resourceType, content, data } = param;
 
-    let storageKey: string;
-    let staticFile: StaticFileModel | null = null;
-    let storageSize = 0;
     let identifier: string;
-    let indexStatus: IndexStatus = 'wait_parse';
-    let contentPreview: string;
+    let staticFile: StaticFileModel | null = null;
 
-    if (param.resourceType === 'weblink') {
-      if (!param.data) {
-        throw new ParamsError('data is required');
-      }
-      const { url, title } = param.data;
-      if (!url) {
-        throw new ParamsError('url is required');
-      }
-      param.title ||= title;
-      param.data.url = normalizeUrl(url, { stripHash: true });
-      identifier = param.data.url;
-    } else if (param.resourceType === 'text') {
-      if (!param.content) {
+    if (resourceType === 'text') {
+      if (!content) {
         throw new ParamsError('content is required for text resource');
       }
-      const md5Hash = crypto
-        .createHash('md5')
-        .update(param.content ?? '')
-        .digest('hex');
-      identifier = `text://${md5Hash}`;
-
-      const cleanedContent = param.content?.replace(/x00/g, '') ?? '';
-
-      if (cleanedContent) {
-        // save text content to object storage
-        storageKey = `resources/${param.resourceId}.txt`;
-        await this.minio.client.putObject(storageKey, cleanedContent);
-        storageSize = (await this.minio.client.statObject(storageKey)).size;
-
-        // skip parse stage, since content is provided
-        indexStatus = 'wait_index';
-        contentPreview = cleanedContent.slice(0, 500);
+      const sha = crypto.createHash('sha256').update(content).digest('hex');
+      identifier = `text://${sha}`;
+    } else if (resourceType === 'weblink') {
+      if (!data?.url) {
+        throw new ParamsError('URL is required for weblink resource');
       }
-    } else if (param.resourceType === 'file') {
+      identifier = normalizeUrl(param.data.url, { stripHash: true });
+    } else if (resourceType === 'file') {
       if (!param.storageKey) {
         throw new ParamsError('storageKey is required for file resource');
       }
       staticFile = await this.prisma.staticFile.findFirst({
-        where: { storageKey: param.storageKey },
+        where: {
+          storageKey: param.storageKey,
+          uid: user.uid,
+          deletedAt: null,
+        },
       });
       if (!staticFile) {
-        throw new StaticFileNotFoundError(`static file ${param.storageKey} not found`);
+        throw new ParamsError(`static file ${param.storageKey} not found`);
       }
-
-      const shasum = crypto.createHash('sha256');
+      const sha = crypto.createHash('sha256');
       const fileStream = await this.minio.client.getObject(staticFile.storageKey);
-      shasum.update(await streamToBuffer(fileStream));
-      identifier = `file:${shasum.digest('hex')}`;
+      sha.update(await streamToBuffer(fileStream));
 
-      param.data = {
-        ...param.data,
-        contentType: staticFile.contentType,
-      };
+      identifier = `file://${sha.digest('hex')}`;
     } else {
       throw new ParamsError('Invalid resource type');
     }
 
+    if (content) {
+      // save content to object storage
+      const storageKey = `resources/${param.resourceId}.txt`;
+      await this.minio.client.putObject(storageKey, param.content);
+      const storageSize = (await this.minio.client.statObject(storageKey)).size;
+
+      return {
+        storageKey,
+        storageSize,
+        identifier,
+        indexStatus: 'wait_index', // skip parsing stage, since content is provided
+        contentPreview: param.content.slice(0, 500),
+      };
+    }
+
+    if (resourceType === 'weblink') {
+      return {
+        identifier,
+        indexStatus: 'wait_parse',
+        metadata: {
+          ...param.data,
+          url: identifier,
+        },
+      };
+    }
+
+    // must be file resource
+    return {
+      identifier,
+      indexStatus: 'wait_parse',
+      staticFile,
+      metadata: {
+        ...param.data,
+        contentType: staticFile.contentType,
+      },
+    };
+  }
+
+  async createResource(
+    user: User,
+    param: UpsertResourceRequest,
+    options?: { checkStorageQuota?: boolean },
+  ) {
+    if (options?.checkStorageQuota) {
+      const usageResult = await this.subscriptionService.checkStorageUsage(user);
+      if (usageResult.available < 1) {
+        throw new StorageQuotaExceeded();
+      }
+    }
+
+    param.resourceId = genResourceID();
+    if (param.content) {
+      param.content = param.content.replace(/x00/g, '');
+    }
+
+    const {
+      identifier,
+      indexStatus,
+      contentPreview,
+      storageKey,
+      storageSize,
+      staticFile,
+      metadata,
+    } = await this.prepareResource(user, param);
+
     const existingResource = await this.prisma.resource.findFirst({
-      where: { uid: user.uid, identifier, deletedAt: null },
+      where: {
+        uid: user.uid,
+        identifier,
+        deletedAt: null,
+      },
     });
     param.resourceId = existingResource ? existingResource.resourceId : genResourceID();
 
@@ -230,22 +263,22 @@ export class KnowledgeService {
         resourceId: param.resourceId,
         identifier,
         resourceType: param.resourceType,
-        meta: JSON.stringify(param.data || {}),
+        meta: JSON.stringify({ ...param.data, ...metadata }),
         contentPreview,
         storageKey,
         storageSize,
         rawFileKey: staticFile?.storageKey,
         uid: user.uid,
-        title: param.title || 'Untitled',
+        title: param.title || '',
         indexStatus,
       },
       update: {
-        meta: JSON.stringify(param.data || {}),
+        meta: JSON.stringify({ ...param.data, ...metadata }),
         contentPreview,
         storageKey,
         storageSize,
         rawFileKey: staticFile?.storageKey,
-        title: param.title || 'Untitled',
+        title: param.title || '',
         indexStatus,
       },
     });
@@ -432,6 +465,8 @@ export class KnowledgeService {
       const fileStream = await this.minio.client.getObject(resource.rawFileKey);
       const fileBuffer = await streamToBuffer(fileStream);
       result = await parser.parse(fileBuffer);
+    } else {
+      throw new Error(`Cannot parse resource ${resourceId} with no content or rawFileKey`);
     }
 
     if (result.error) {

apps/extension/src/components/content-clipper/index.scss

@@ -0,0 +1,9 @@
+.content-clipper-collapse {
+  .ant-collapse-header {
+    padding: 0;
+  }
+
+  .ant-collapse-content {
+    padding: 0;
+  }
+}

apps/extension/src/components/content-clipper/index.tsx

@@ -1,5 +1,5 @@
 import React, { useCallback, useState, useEffect } from 'react';
-import { Button, Divider, Input, message, Tooltip } from 'antd';
+import { Button, Divider, Input, message, Tooltip, Form, Collapse } from 'antd';
 import { IconDelete, IconPaste } from '@arco-design/web-react/icon';
 import { HiOutlineDocumentDownload } from 'react-icons/hi';
 import { useTranslation } from 'react-i18next';
@@ -14,6 +14,7 @@ import { BackgroundMessage } from '@refly/common-types';
 import { getRuntime } from '@refly/utils/env';
 
 const { TextArea } = Input;
+const { Panel } = Collapse;
 
 interface ContentClipperProps {
   className?: string;
@@ -174,17 +175,55 @@ export const ContentClipper: React.FC<ContentClipperProps> = ({ className, onSav
     setPageInfo((prev) => ({ ...prev, content: e.target.value }));
   }, []);
 
+  // Handle title change
+  const handleTitleChange = useCallback((e: React.ChangeEvent<HTMLTextAreaElement>) => {
+    setPageInfo((prev) => ({ ...prev, title: e.target.value }));
+  }, []);
+
+  // Handle URL change
+  const handleUrlChange = useCallback((e: React.ChangeEvent<HTMLTextAreaElement>) => {
+    setPageInfo((prev) => ({ ...prev, url: e.target.value }));
+  }, []);
+
   return (
     <div className={`flex flex-col gap-4 p-0 ${className}`}>
       <div className="flex flex-col gap-2">
-        <TextArea
-          placeholder={t('translation:extension.webClipper.placeholder.enterOrClipContent')}
-          value={pageInfo.content}
-          onChange={handleContentChange}
-          onKeyDown={handleKeyDown}
-          autoSize={{ minRows: 10, maxRows: 10 }}
-          className="w-full resize-none"
-        />
+        <Form layout="vertical" className="content-clipper-form">
+          <TextArea
+            placeholder={t('translation:extension.webClipper.placeholder.enterOrClipContent')}
+            value={pageInfo.content}
+            onChange={handleContentChange}
+            onKeyDown={handleKeyDown}
+            autoSize={{ minRows: 10, maxRows: 10 }}
+            className="w-full resize-none"
+          />
+          <Collapse ghost className="m-0 p-0 content-clipper-collapse">
+            <Panel
+              header={t('translation:extension.webClipper.placeholder.metadata')}
+              key="metadata"
+            >
+              <Form.Item label={t('translation:extension.webClipper.placeholder.title')}>
+                <TextArea
+                  value={pageInfo.title}
+                  onChange={handleTitleChange}
+                  className="content-clipper-input"
+                  rows={1}
+                  placeholder={t('translation:extension.webClipper.placeholder.enterTitle')}
+                />
+              </Form.Item>
+              <Form.Item label={t('translation:extension.webClipper.placeholder.url')}>
+                <TextArea
+                  value={pageInfo.url}
+                  onChange={handleUrlChange}
+                  className="content-clipper-input"
+                  rows={1}
+                  placeholder={t('translation:extension.webClipper.placeholder.enterUrl')}
+                />
+              </Form.Item>
+            </Panel>
+          </Collapse>
+        </Form>
+
         <div className="flex flex-row justify-end gap-2">
           <div className="flex flex-row gap-2">
             {pageInfo.content && (