Merge pull request #429 from refly-ai/fix/docker-deploy-issues

fix(docker): deployment issues with minio and elasticsearch

.github/workflows/build-es-image.yml

@@ -0,0 +1,35 @@
+name: Build Elasticsearch Image
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build Elasticsearch Image
+    runs-on: ubuntu-latest
+    if: github.repository == 'refly-ai/refly'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: reflyai
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./deploy/docker/elasticsearch/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            reflyai/elasticsearch:7.10.2

apps/api/.env.example

@@ -56,19 +56,19 @@ ELASTICSEARCH_USERNAME=
 ELASTICSEARCH_PASSWORD=
 
 # MinIO Configuration
-MINIO_INTERNAL_ENDPOINT=localhost
-MINIO_INTERNAL_PORT=9000
-MINIO_INTERNAL_USE_SSL=false
-MINIO_INTERNAL_ACCESS_KEY=minioadmin
-MINIO_INTERNAL_SECRET_KEY=minioadmin
-MINIO_INTERNAL_BUCKET=refly-weblink
-
-MINIO_EXTERNAL_ENDPOINT=localhost
-MINIO_EXTERNAL_PORT=9000
-MINIO_EXTERNAL_USE_SSL=false
-MINIO_EXTERNAL_ACCESS_KEY=minioadmin
-MINIO_EXTERNAL_SECRET_KEY=minioadmin
-MINIO_EXTERNAL_BUCKET=refly-weblink
+MINIO_INTERNAL_ENDPOINT=
+MINIO_INTERNAL_PORT=
+MINIO_INTERNAL_USE_SSL=
+MINIO_INTERNAL_ACCESS_KEY=
+MINIO_INTERNAL_SECRET_KEY=
+MINIO_INTERNAL_BUCKET=
+
+MINIO_EXTERNAL_ENDPOINT=
+MINIO_EXTERNAL_PORT=
+MINIO_EXTERNAL_USE_SSL=
+MINIO_EXTERNAL_ACCESS_KEY=
+MINIO_EXTERNAL_SECRET_KEY=
+MINIO_EXTERNAL_BUCKET=
 
 # Authentication Configuration
 AUTH_SKIP_VERIFICATION=true

apps/api/src/main.ts

@@ -29,7 +29,7 @@ Sentry.init({
 async function bootstrap() {
   const app = await NestFactory.create<NestExpressApplication>(AppModule, {
     rawBody: true,
-    bufferLogs: true,
+    bufferLogs: false,
   });
   app.useLogger(app.get(Logger));
 

deploy/docker/.env.example

@@ -56,19 +56,19 @@ ELASTICSEARCH_USERNAME=
 ELASTICSEARCH_PASSWORD=
 
 # MinIO Configuration
-MINIO_INTERNAL_ENDPOINT=localhost
-MINIO_INTERNAL_PORT=9000
-MINIO_INTERNAL_USE_SSL=false
-MINIO_INTERNAL_ACCESS_KEY=minioadmin
-MINIO_INTERNAL_SECRET_KEY=minioadmin
-MINIO_INTERNAL_BUCKET=refly-weblink
-
-MINIO_EXTERNAL_ENDPOINT=localhost
-MINIO_EXTERNAL_PORT=9000
-MINIO_EXTERNAL_USE_SSL=false
-MINIO_EXTERNAL_ACCESS_KEY=minioadmin
-MINIO_EXTERNAL_SECRET_KEY=minioadmin
-MINIO_EXTERNAL_BUCKET=refly-weblink
+MINIO_INTERNAL_ENDPOINT=
+MINIO_INTERNAL_PORT=
+MINIO_INTERNAL_USE_SSL=
+MINIO_INTERNAL_ACCESS_KEY=
+MINIO_INTERNAL_SECRET_KEY=
+MINIO_INTERNAL_BUCKET=
+
+MINIO_EXTERNAL_ENDPOINT=
+MINIO_EXTERNAL_PORT=
+MINIO_EXTERNAL_USE_SSL=
+MINIO_EXTERNAL_ACCESS_KEY=
+MINIO_EXTERNAL_SECRET_KEY=
+MINIO_EXTERNAL_BUCKET=
 
 # Authentication Configuration
 AUTH_SKIP_VERIFICATION=true

deploy/docker/docker-compose.middleware.yml

@@ -44,7 +44,7 @@ services:
     restart: always
   elasticsearch:
     container_name: refly_elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
+    image: reflyai/elasticsearch:7.10.2
     ports:
       - 9200:9200
     environment:

deploy/docker/docker-compose.yml

@@ -20,7 +20,8 @@ services:
     env_file:
       - .env
     environment:
-      - MINIO_ENDPOINT=http://minio
+      - MINIO_INTERNAL_ENDPOINT=minio
+      - MINIO_EXTERNAL_ENDPOINT=minio
       - REDIS_HOST=redis
       - DATABASE_URL=postgresql://refly:test@db:5432/refly?schema=refly
       - QDRANT_HOST=qdrant

deploy/docker/elasticsearch/Dockerfile

@@ -0,0 +1,4 @@
+FROM docker.elastic.co/elasticsearch/elasticsearch:7.10.2
+
+# Install ICU Analysis plugin
+RUN elasticsearch-plugin install analysis-icu --batch