fix: don't block migration if file has already been uploaded to S3

SocialGouv · Oct 10, 2024 · 1839e3c · 1839e3c
1 parent 62ddc6c
commit 1839e3c
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 14 deletions.
diff --git a/.talismanrc b/.talismanrc
@@ -246,7 +246,7 @@ fileignoreconfig:
 - filename: packages/migrations/src/migrations/20240710125537_eig.js
   checksum: c358c5f8cf7a408173e9ff8d273f05c8e1a193c3369ec2389bfd713ff1bab716
 - filename: packages/migrations/src/migrations/20240918085236_migrate_files_to_minio.js
-  checksum: e6941cda67cdcf254719bc7cc5e1fde23365c22b62116fd49dad74d466d9f9f4
+  checksum: 697b937bde70ecf883bfc42137bf7bb34924dc860546cc0fc525d75c32f252b2
 - filename: packages/shared/src/components/Chat.vue
   checksum: f2dbbf72bf098c7abd2c3aee230d220f5a5a106952883c0e116eb49f4f9d4db7
 - filename: packages/shared/src/components/PasswordInput.vue

diff --git a/packages/migrations/src/migrations/20240918085236_migrate_files_to_minio.js b/packages/migrations/src/migrations/20240918085236_migrate_files_to_minio.js
@@ -1,4 +1,8 @@
-const { S3Client, PutObjectCommand } = require("@aws-sdk/client-s3");
+const {
+  S3Client,
+  PutObjectCommand,
+  HeadObjectCommand,
+} = require("@aws-sdk/client-s3");
 
 const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME;
 const S3_BUCKET_ROOT_DIR = process.env.S3_BUCKET_ROOT_DIR;
@@ -27,24 +31,48 @@ exports.up = (knex) => {
     .from("documents")
     .then(async (rows) => {
       for (const row of rows) {
+        const objectKey = `${S3_BUCKET_ROOT_DIR}/${row.uuid}.pdf`;
+
         try {
+          // Check if the file already exists
           await s3Client.send(
-            new PutObjectCommand({
-              Body: row.file,
+            new HeadObjectCommand({
               Bucket: S3_BUCKET_NAME,
-              Key: `${S3_BUCKET_ROOT_DIR}/${row.uuid}.pdf`,
-              Metadata: {
-                category: String(row.category),
-                created_at: String(row.created_at),
-                mimetype: String(row.mime_type),
-                originalname: String(row.filename),
-              },
+              Key: objectKey,
             }),
           );
-          console.log(`Uploaded ${S3_BUCKET_ROOT_DIR}/${row.uuid}`);
+
+          // If the file exists, log it and skip the upload
+          console.log(`File ${objectKey} already exists. Skipping upload.`);
+          continue;
         } catch (err) {
-          console.error(`Failed to upload ${row.uuid}:`, err);
-          throw err;
+          // If a 404 error occurs, the file doesn't exist, proceed with upload
+          if (err.name === "NotFound") {
+            try {
+              // Upload the file since it doesn't exist
+              await s3Client.send(
+                new PutObjectCommand({
+                  Body: row.file,
+                  Bucket: S3_BUCKET_NAME,
+                  Key: objectKey,
+                  Metadata: {
+                    category: String(row.category),
+                    created_at: String(row.created_at),
+                    mimetype: String(row.mime_type),
+                    originalname: String(row.filename),
+                  },
+                }),
+              );
+              console.log(`Uploaded ${objectKey}`);
+            } catch (uploadErr) {
+              console.error(`Failed to upload ${objectKey}:`, uploadErr);
+              throw uploadErr;
+            }
+          } else {
+            // If the error isn't a 404, throw it to handle other failures
+            console.error(`Error checking existence of ${objectKey}:`, err);
+            throw err;
+          }
         }
       }
     })