Merge branch 'main' into feat/isomorphic-copilot-for-extension

.gitattributes

@@ -0,0 +1,14 @@
+# Don't allow people to merge changes to these generated files, because the result
+# may be invalid.  You need to run "rush update" again.
+pnpm-lock.yaml               merge=text
+shrinkwrap.yaml              merge=binary
+npm-shrinkwrap.json          merge=binary
+yarn.lock                    merge=binary
+
+# Rush's JSON config files use JavaScript-style code comments.  The rule below prevents pedantic
+# syntax highlighters such as GitHub's from highlighting these comments as errors.  Your text editor
+# may also require a special configuration to allow comments in JSON.
+#
+# For more information, see this issue: https://github.com/microsoft/rushstack/issues/1088
+#
+*.json                       linguist-language=JSON-with-Comments

.github/workflows/deploy-reflyd-base.yml → .github/workflows/deploy-api-base.yml

@@ -1,4 +1,4 @@
-name: Deploy Reflyd
+name: Deploy API
 
 on:
   workflow_call:
@@ -55,7 +55,7 @@ jobs:
         env:
           IMAGE_TAG: ${{ github.sha }}
         run: |
-          docker build -t $ECR_REPOSITORY:$IMAGE_TAG reflyd
+          docker build -t $ECR_REPOSITORY:$IMAGE_TAG apps/api
           docker push $ECR_REPOSITORY:$IMAGE_TAG
           echo "image=$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT
 

.github/workflows/deploy-reflyd-prod.yml → .github/workflows/deploy-api-prod.yml

@@ -1,16 +1,16 @@
-name: Deploy Reflyd To Prod
+name: Deploy API To Prod
 
 on:
   push:
     branches:
       - release
     paths:
-      - "reflyd/**"
+      - 'apps/api/**'
   workflow_dispatch:
 
 jobs:
   deploy-prod:
-    uses: ./.github/workflows/deploy-reflyd-base.yml
+    uses: ./.github/workflows/deploy-api-base.yml
     with:
       environment: production
       ecs-cluster: Prod

.github/workflows/deploy-reflyd-staging.yml → .github/workflows/deploy-api-staging.yml

@@ -1,16 +1,16 @@
-name: Deploy Reflyd To Staging
+name: Deploy API To Staging
 
 on:
   push:
     branches-ignore:
       - release
     paths:
-      - "reflyd/**"
+      - 'apps/api/**'
   workflow_dispatch:
 
 jobs:
   deploy-staging:
-    uses: ./.github/workflows/deploy-reflyd-base.yml
+    uses: ./.github/workflows/deploy-api-base.yml
     with:
       environment: staging
       ecs-cluster: Prod

.github/workflows/deploy-web-base.yml

@@ -32,12 +32,11 @@ jobs:
       - name: Use Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: "20.x"
-          cache: "yarn"
-          cache-dependency-path: web/yarn.lock
+          node-version: '20.x'
+          cache: 'pnpm'
 
       - name: Build
-        run: cd web && yarn --frozen-lockfile && yarn build
+        run: pnpm install && pnpm --filter '@refly/web' build
 
       - name: Publish to Cloudflare Pages
         uses: cloudflare/pages-action@v1
@@ -46,5 +45,5 @@ jobs:
           accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
           projectName: ${{ inputs.project-name }}
           branch: main # always main, to ensure production deployment
-          workingDirectory: web
+          workingDirectory: apps/web
           directory: dist

.github/workflows/deploy-web-prod.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - release
     paths:
-      - "web/**"
+      - 'apps/web/**'
   workflow_dispatch:
 
 jobs:

.github/workflows/deploy-web-staging.yml

@@ -5,7 +5,7 @@ on:
     branches-ignore:
       - release
     paths:
-      - "web/**"
+      - 'apps/web/**'
   workflow_dispatch:
 
 jobs:

.gitignore

@@ -1,10 +1,16 @@
-.venv
-.mypy_cache
-__pycache__
 .yarn
 .env*
 *.env
 **/node_modules/**
 .DS_Store
 
-qdrant_storage/**/**
+dist
+
+# Rush temporary files
+common/deploy/
+common/temp/
+common/autoinstallers/*/.npmrc
+**/.rush/temp/
+*.lock
+*.log
+*.chunks.jsonl

.prettierrc.yaml

@@ -0,0 +1,5 @@
+tabWidth: 2
+trailingComma: all
+singleQuote: true
+semi: true
+printWidth: 120

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+  "files.associations": {
+    "*.json": "jsonc"
+  },
+  "typescript.preferences.importModuleSpecifier": "relative"
+}

reflyd/package.json → apps/api/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "api",
+  "name": "@refly/api",
   "version": "0.0.1",
   "description": "",
   "author": "",
@@ -29,22 +29,23 @@
   "dependencies": {
     "@dqbd/tiktoken": "1.0.7",
     "@esm2cjs/normalize-url": "^8.0.0",
-    "@langchain/community": "^0.0.32",
-    "@langchain/openai": "^0.0.14",
+    "@langchain/community": "~0.2.3",
+    "@langchain/openai": "~0.0.34",
     "@mozilla/readability": "^0.5.0",
     "@nestjs/bull": "^10.1.0",
     "@nestjs/common": "^9.0.0",
     "@nestjs/config": "~2.3.2",
     "@nestjs/core": "^9.0.0",
     "@nestjs/jwt": "~10.0.3",
-    "@nestjs/microservices": "^9.4.3",
     "@nestjs/passport": "~9.0.3",
     "@nestjs/platform-express": "^9.0.0",
     "@nestjs/swagger": "^7.3.0",
     "@opentelemetry/api": "^1.8.0",
     "@opentelemetry/auto-instrumentations-node": "^0.44.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
+    "@opentelemetry/resources": "^1.24.1",
     "@opentelemetry/sdk-node": "^0.50.0",
+    "@opentelemetry/semantic-conventions": "^1.24.1",
     "@paralleldrive/cuid2": "^2.2.2",
     "@prisma/client": "5",
     "@prisma/instrumentation": "^5.12.1",
@@ -61,10 +62,10 @@
     "hbs": "^4.2.0",
     "helmet": "^7.1.0",
     "ioredis": "^5.4.1",
-    "langchain": "^0.1.21",
-    "lodash.omit": "^4.5.0",
+    "langchain": "~0.2.2",
+    "lodash": "^4.17.21",
     "lru-cache": "^10.2.0",
-    "minio": "^7.1.3",
+    "minio": "7.1.3",
     "nestjs-pino": "^4.0.0",
     "node-fetch": "^2.0.0",
     "passport": "~0.6.0",
@@ -81,7 +82,10 @@
     "turndown": "^7.1.3",
     "turndown-plugin-gfm": "^1.0.2",
     "uuid": "^9.0.1",
-    "zod": "^3.22.4"
+    "zod": "^3.22.4",
+    "@refly/constants": "workspace:*",
+    "@refly/schema": "workspace:*",
+    "@langchain/core": "~0.2.2"
   },
   "devDependencies": {
     "@nestjs/cli": "^9.0.0",
@@ -90,7 +94,7 @@
     "@types/cookie-parser": "~1.4.3",
     "@types/express": "^4.17.13",
     "@types/jest": "29.5.1",
-    "@types/lodash.omit": "^4.5.9",
+    "@types/lodash": "^4.17.4",
     "@types/node": "18.16.12",
     "@types/passport": "~1.0.12",
     "@types/passport-github2": "~1.2.5",
@@ -100,20 +104,17 @@
     "@types/pg": "^8.11.4",
     "@types/supertest": "^2.0.11",
     "@types/turndown": "^5.0.4",
-    "@typescript-eslint/eslint-plugin": "^5.0.0",
-    "@typescript-eslint/parser": "^5.0.0",
+    "@typescript-eslint/eslint-plugin": "~7.11.0",
+    "@typescript-eslint/parser": "~7.11.0",
     "eslint": "^8.0.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-prettier": "^4.0.0",
-    "grpc-tools": "^1.12.4",
-    "grpc_tools_node_protoc_ts": "^5.3.3",
     "jest": "29.5.0",
     "prettier": "^2.3.2",
     "prisma": "5",
     "source-map-support": "^0.5.20",
     "supertest": "^6.1.3",
     "ts-jest": "29.1.0",
-    "ts-loader": "^9.2.3",
     "ts-node": "^10.0.0",
     "ts-proto": "^1.172.0",
     "tsconfig-paths": "4.2.0",

reflyd/src/aigc/aigc.controller.ts → apps/api/src/aigc/aigc.controller.ts

@@ -1,5 +1,5 @@
 import { Controller, Logger, Get, Post, UseGuards, Request, Query, Param } from '@nestjs/common';
-import omit from 'lodash.omit';
+import { omit } from 'lodash';
 import { ApiBody, ApiResponse } from '@nestjs/swagger';
 
 import { JwtAuthGuard } from '../auth/guard/jwt-auth.guard';

reflyd/src/aigc/aigc.service.ts → apps/api/src/aigc/aigc.service.ts

@@ -1,6 +1,6 @@
 import { Logger, Injectable } from '@nestjs/common';
 import { Document } from '@langchain/core/documents';
-import omit from 'lodash.omit';
+import { omit } from 'lodash';
 
 import { LlmService } from '../llm/llm.service';
 import { PrismaService } from '../common/prisma.service';

reflyd/src/auth/auth.module.ts → apps/api/src/auth/auth.module.ts

@@ -35,13 +35,7 @@ import { GoogleOauthStrategy } from './strategy/google-oauth.strategy';
       inject: [ConfigService],
     }),
   ],
-  providers: [
-    AuthService,
-    LocalStrategy,
-    JwtStrategy,
-    GithubOauthStrategy,
-    GoogleOauthStrategy,
-  ],
+  providers: [AuthService, LocalStrategy, JwtStrategy, GithubOauthStrategy, GoogleOauthStrategy],
   exports: [AuthService],
   controllers: [AuthController],
 })

reflyd/src/common/minio.service.ts → apps/api/src/common/minio.service.ts

@@ -1,7 +1,7 @@
 import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { Client } from 'minio';
-import { streamToBuffer } from 'src/utils';
+import { streamToBuffer } from '../utils';
 
 @Injectable()
 export class MinioService implements OnModuleInit {

reflyd/src/conversation/conversation.controller.ts → apps/api/src/conversation/conversation.controller.ts

@@ -21,7 +21,7 @@ import {
 import { ApiParam, ApiResponse } from '@nestjs/swagger';
 import { JwtAuthGuard } from '../auth/guard/jwt-auth.guard';
 import { ConversationService } from './conversation.service';
-import { TASK_TYPE, type Task } from './conversation.dto';
+import { TASK_TYPE, Task } from './conversation.dto';
 import { Conversation } from '@prisma/client';
 
 @Controller('conversation')

reflyd/src/conversation/conversation.service.ts → apps/api/src/conversation/conversation.service.ts

@@ -16,7 +16,7 @@ import { createLLMChatMessage } from '../llm/schema';
 import { LlmService } from '../llm/llm.service';
 import { WeblinkService } from '../weblink/weblink.service';
 import { IterableReadableStream } from '@langchain/core/dist/utils/stream';
-import { BaseMessageChunk } from 'langchain/schema';
+import { BaseMessageChunk } from '@langchain/core/messages';
 import { genConvID } from '../utils/id';
 import { AigcService } from '../aigc/aigc.service';
 

reflyd/src/knowledge/knowledge.controller.ts → apps/api/src/knowledge/knowledge.controller.ts

@@ -13,8 +13,6 @@ import {
   UnauthorizedException,
 } from '@nestjs/common';
 import { ApiResponse } from '@nestjs/swagger';
-import { KnowledgeService } from './knowledge.service';
-import { JwtAuthGuard } from '../auth/guard/jwt-auth.guard';
 import {
   UpsertCollectionRequest,
   UpsertCollectionResponse,
@@ -26,9 +24,12 @@ import {
   GetResourceDetailResponse,
   DeleteCollectionRequest,
   DeleteResourceRequest,
-  convertResourcePoToListItem,
   CollectionListItem,
-} from './knowledge.dto';
+} from '@refly/schema';
+import { KnowledgeService } from './knowledge.service';
+import { JwtAuthGuard } from '../auth/guard/jwt-auth.guard';
+
+import { convertResourcePoToListItem } from './knowledge.dto';
 
 @Controller('knowledge')
 export class KnowledgeController {

apps/api/src/knowledge/knowledge.dto.ts

@@ -0,0 +1,7 @@
+import _ from 'lodash';
+import { Resource } from '@prisma/client';
+import { ResourceListItem } from '@refly/schema';
+
+export const convertResourcePoToListItem = (resource: Resource): ResourceListItem => {
+  return _.omit(resource, ['id', 'userId', 'deletedAt']);
+};

reflyd/src/knowledge/knowledge.processor.ts → apps/api/src/knowledge/knowledge.processor.ts

@@ -2,9 +2,9 @@ import { Process, Processor } from '@nestjs/bull';
 import { Logger } from '@nestjs/common';
 import { Job } from 'bull';
 
+import { UpsertResourceRequest } from '@refly/schema';
 import { KnowledgeService } from './knowledge.service';
 import { QUEUE_RESOURCE, CHANNEL_FINALIZE_RESOURCE } from '../utils/const';
-import { UpsertResourceRequest } from './knowledge.dto';
 
 @Processor(QUEUE_RESOURCE)
 export class ResourceProcessor {

reflyd/src/knowledge/knowledge.service.ts → apps/api/src/knowledge/knowledge.service.ts

@@ -15,15 +15,15 @@ import {
   ResourceDetail,
   WeblinkMeta,
   CollectionListItem,
-} from './knowledge.dto';
+} from '@refly/schema';
 import {
   CHANNEL_FINALIZE_RESOURCE,
   QUEUE_RESOURCE,
   cleanMarkdownForIngest,
   genCollectionID,
   genResourceID,
   genResourceUuid,
-} from 'src/utils';
+} from '../utils';
 
 @Injectable()
 export class KnowledgeService {