Fix CI (#3)

* Improved directory upload handling

* Increase waitUntil timeOut

* test utilities: introduce waiting for modified timestamp sync

* add concurrently as a dependency

so that dependencies can be cached properly on CI (lockfile can't be changed)

Author: s-h-a-d-o-w

.github/workflows/ci.yml

@@ -4,6 +4,9 @@ on:
   push:
     branches:
       - master
+  pull_request:
+    branches:
+      - master
 
 jobs:
   build:
@@ -24,9 +27,7 @@ jobs:
           node-version: 22
           cache: pnpm
       - name: Install dependencies
-        run: |
-          pnpm install
-          pnpm add concurrently
+        run: pnpm install
       - name: Test
         run: pnpm concurrently "pnpm test" "pnpm typecheck" "pnpm lint"
         env:

client/tests/e2e.spec.mts

@@ -1,5 +1,5 @@
 import { fileExists } from "@s3-smart-sync/shared/fileExists.js";
-import { mkdir, readFile, rm, stat } from "node:fs/promises";
+import { readFile, rm, stat } from "node:fs/promises";
 import { join } from "node:path";
 import {
   UNIGNORE_DURATION,
@@ -10,6 +10,7 @@ import {
   cleanupS3,
   clientLogs,
   createClientDirectories,
+  createDirectory,
   createFile,
   list,
   pause,
@@ -32,14 +33,27 @@ globalThis.it = (name: string, fn: () => Promise<void>, timeout?: number) => {
   originalIt(
     name,
     async function () {
-      process.stdout.write(`===========================================\n`);
-      process.stdout.write(`${name}\n`);
-      process.stdout.write(`===========================================\n`);
+      process.stdout.write(
+        `===============================================================================\n`,
+      );
+      process.stdout.write(` 🧪 ${name}\n`);
+      process.stdout.write(
+        `===============================================================================\n`,
+      );
       return await fn();
     },
     timeout,
   );
 };
+// @ts-expect-error
+// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+globalThis.it.only = (
+  name: string,
+  fn: () => Promise<void>,
+  timeout?: number,
+) => {
+  originalIt.only(name, fn, timeout);
+};
 
 describe("E2E Tests", () => {
   beforeAll(async () => {
@@ -118,7 +132,9 @@ describe("E2E Tests", () => {
     await stopClients([1]);
     await Promise.all(
       Object.entries(TEST_FILES).map(([key, content]) =>
-        createFile(0, key, content),
+        key.endsWith("/")
+          ? createDirectory(0, key as `${string}/`)
+          : createFile(0, key, content),
       ),
     );
     await startClients([1]);
@@ -177,11 +193,7 @@ describe("E2E Tests", () => {
     await pause(WATCHER_DEBOUNCE_DURATION + 300);
     await sendSnsMessage("file-then-directory", "delete");
 
-    await mkdir(join(clientDirectories[0]!, "file-then-directory"));
-    // First, the debounced upload. Then we have to wait for the upload to actually have finished
-    await pause(WATCHER_DEBOUNCE_DURATION + 300);
-    await sendSnsMessage("file-then-directory/", "put");
-
+    await createDirectory(0, "file-then-directory/");
     await waitUntil(async () =>
       (
         await stat(join(clientDirectories[1]!, "file-then-directory"))
@@ -190,10 +202,7 @@ describe("E2E Tests", () => {
   });
 
   it("should handle replacing an empty directory with a file", async () => {
-    await mkdir(join(clientDirectories[0]!, "directory-then-file"));
-    // First, the debounced upload. Then we have to wait for the upload to actually have finished
-    await pause(WATCHER_DEBOUNCE_DURATION + 300);
-    await sendSnsMessage("directory-then-file/", "put");
+    await createDirectory(0, "directory-then-file/");
     await waitUntil(async () =>
       (
         await stat(join(clientDirectories[1]!, "directory-then-file"))
@@ -203,7 +212,7 @@ describe("E2E Tests", () => {
     await rm(join(clientDirectories[0]!, "directory-then-file"), {
       recursive: true,
     });
-    await pause(WATCHER_DEBOUNCE_DURATION + 300);
+    await pause(WATCHER_DEBOUNCE_DURATION + 1000);
     await waitUntil(async () => {
       const { Contents } = await list("directory-then-file/");
       return Contents === undefined;

client/tests/utilities.ts

@@ -9,7 +9,7 @@ import {
 import { Upload } from "@aws-sdk/lib-storage";
 import { logger } from "@s3-smart-sync/shared/logger.js";
 import { spawn } from "child_process";
-import { mkdir, rm, writeFile, readdir } from "node:fs/promises";
+import { mkdir, readdir, rm, stat, writeFile } from "node:fs/promises";
 import path, { join } from "node:path";
 import {
   ACCESS_KEY,
@@ -81,6 +81,13 @@ export async function createClientDirectories<T extends readonly number[]>(
   ) as Record<T[number], string>;
 }
 
+/**
+ * Includes sending SNS message
+ */
+export async function createDirectory(id: number, key: `${string}/`) {
+  await createFile(id, key, "");
+}
+
 /**
  * Includes sending SNS message
  */
@@ -93,27 +100,33 @@ export async function createFile(id: number, key: string, content: string) {
     await writeFile(join(clientDirectory, key), content);
   }
 
-  // We have to check content in case the file already existed
+  let lastModified: Date | undefined;
   await waitUntil(async () => {
-    if (key.endsWith("/")) {
-      await s3Client.send(
-        new GetObjectCommand({
-          Bucket: S3_BUCKET,
-          Key: key,
-        }),
-      );
-    } else {
-      const { Body } = await s3Client.send(
-        new GetObjectCommand({
-          Bucket: S3_BUCKET,
-          Key: key,
-        }),
-      );
-      const actualContent = await Body?.transformToString();
-      return actualContent === content;
-    }
+    const { Body, LastModified } = await s3Client.send(
+      new GetObjectCommand({
+        Bucket: S3_BUCKET,
+        Key: key,
+      }),
+    );
+    lastModified = LastModified;
+
+    // We have to check content in case the file already existed
+    const actualContent = await Body?.transformToString();
+    return actualContent === content;
   });
 
+  // Wait for modified timestamp syncing
+  if (lastModified) {
+    await waitUntil(async () => {
+      return (
+        (await stat(join(clientDirectory, key))).mtime.valueOf() ===
+        lastModified!.valueOf()
+      );
+    });
+  } else {
+    throw new Error("No last modified info for " + key);
+  }
+
   await sendSnsMessage(key, "put");
 }
 
@@ -272,7 +285,7 @@ export async function waitUntil(
   fn: () => unknown,
   {
     interval = 200,
-    timeout = 3000,
+    timeout = 5000,
   }: { interval?: number; timeout?: number } = {},
 ) {
   const startTime = Date.now();

package.json

@@ -26,6 +26,7 @@
     "@eslint/js": "^9.21.0",
     "@types/eslint__js": "^8.42.3",
     "@types/node": "^20.17.19",
+    "concurrently": "^9.1.2",
     "eslint": "^9.21.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-prettier": "^5.2.3",