feat: Add configurable etag algorithm for file backend

supabase · Jan 14, 2025 · da7aff6 · da7aff6
1 parent 8e1bb78
commit da7aff6
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 7 deletions.
diff --git a/.env.sample b/.env.sample
@@ -100,7 +100,7 @@ AWS_SECRET_ACCESS_KEY=secret1234
 # File Backend
 #######################################
 STORAGE_FILE_BACKEND_PATH=./data
-
+STORAGE_FILE_ETAG_ALGORITHM=md5
 
 #######################################
 # Image Transformation

diff --git a/src/config.ts b/src/config.ts
@@ -20,6 +20,7 @@ type StorageConfigType = {
   uploadFileSizeLimit: number
   uploadFileSizeLimitStandard?: number
   storageFilePath?: string
+  storageFileEtagAlgorithm: 'mtime' | 'md5'
   storageS3MaxSockets: number
   storageS3Bucket: string
   storageS3Endpoint?: string
@@ -273,6 +274,7 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType {
       'STORAGE_FILE_BACKEND_PATH',
       'FILE_STORAGE_BACKEND_PATH'
     ),
+    storageFileEtagAlgorithm: getOptionalConfigFromEnv('STORAGE_FILE_ETAG_ALGORITHM') || 'mtime',
 
     // Storage - S3
     storageS3MaxSockets: parseInt(

diff --git a/src/storage/backend/file.ts b/src/storage/backend/file.ts
@@ -45,17 +45,25 @@ const METADATA_ATTR_KEYS = {
 export class FileBackend implements StorageBackendAdapter {
   client = null
   filePath: string
+  etagAlgorithm: 'mtime' | 'md5'
 
   constructor() {
-    const { storageFilePath } = getConfig()
+    const { storageFilePath, storageFileEtagAlgorithm } = getConfig()
     if (!storageFilePath) {
       throw new Error('FILE_STORAGE_BACKEND_PATH env variable not set')
     }
     this.filePath = storageFilePath
+    this.etagAlgorithm = storageFileEtagAlgorithm
   }
 
-  private etag(stats: fs.Stats): string {
-    return `"${stats.mtimeMs.toString(16)}-${stats.size.toString(16)}"`
+  private async etag(file: string, stats: fs.Stats): Promise<string> {
+    if (this.etagAlgorithm === 'md5') {
+      const checksum = await fileChecksum(file)
+      return `"${checksum}"`
+    } else if (this.etagAlgorithm === 'mtime') {
+      return `"${stats.mtimeMs.toString(16)}-${stats.size.toString(16)}"`
+    }
+    throw new Error('FILE_STORAGE_ETAG_ALGORITHM env variable must be either "mtime" or "md5"')
   }
 
   /**
@@ -74,7 +82,7 @@ export class FileBackend implements StorageBackendAdapter {
     // 'Range: bytes=#######-######
     const file = path.resolve(this.filePath, withOptionalVersion(`${bucketName}/${key}`, version))
     const data = await fs.stat(file)
-    const eTag = this.etag(data)
+    const eTag = await this.etag(file, data)
     const fileSize = data.size
     const { cacheControl, contentType } = await this.getFileMetadata(file)
     const lastModified = new Date(0)
@@ -209,7 +217,7 @@ export class FileBackend implements StorageBackendAdapter {
     await this.setFileMetadata(destFile, Object.assign({}, originalMetadata, metadata))
 
     const fileStat = await fs.lstat(destFile)
-    const eTag = this.etag(fileStat)
+    const eTag = await this.etag(destFile, fileStat)
 
     return {
       httpStatusCode: 200,
@@ -256,7 +264,7 @@ export class FileBackend implements StorageBackendAdapter {
     const { cacheControl, contentType } = await this.getFileMetadata(file)
     const lastModified = new Date(0)
     lastModified.setUTCMilliseconds(data.mtimeMs)
-    const eTag = this.etag(data)
+    const eTag = await this.etag(file, data)
 
     return {
       httpStatusCode: 200,