done

Author: tripolskypetr

.env.example

@@ -1,3 +1,4 @@
 APPWRITE_ENDPOINT=https://cloud.appwrite.io/v1
 APPWRITE_PROJECT_ID=64b53d0c41fcf5093b12
 APPWRITE_API_KEY=****
+APPWRITE_SELF_SIGNED=1

README.md

@@ -2,19 +2,23 @@
 
 > Minimalistic Appwrite schema dumper with data backup and restore features
 
+This backup tool will generate query for each document in AppWrite database and save them as a json files on a hard drive. That means It can handle as much documents as you need. Also there is a script to run AppWrite in Docker on localhost so you can test your backup.
+
 <a href="https://cloud.appwrite.io/card/64b53d046c81edba0b1a">
 	<img width="350" src="https://cloud.appwrite.io/v1/cards/cloud?userId=64b53d046c81edba0b1a" alt="Appwrite Cloud Card" />
 </a>
 
+Got a question? Feel free to [ask It in issues](https://github.com/react-declarative/appwrite-backup-tool/issues), I need traffic
+
 ## Setup
 
 1. Install [Appwrite CLI](https://appwrite.io/docs/command-line) and login
 
 > Windows
 
-```cmd
+```powershell
 npm install -g appwrite-cli
-Set-ExecutionPolicy RemoteSigned
+Set-ExecutionPolicy RemoteSigned # In PowerShell as Administrator
 appwrite client --endpoint https://cloud.appwrite.io/v1
 appwrite login
 ```
@@ -33,6 +37,7 @@ appwrite login
 APPWRITE_ENDPOINT=https://cloud.appwrite.io/v1
 APPWRITE_PROJECT_ID=64b53d0c41fcf5093b12
 APPWRITE_API_KEY=****
+APPWRITE_SELF_SIGNED=1
 ```
 
 ## Usage
@@ -44,6 +49,7 @@ APPWRITE_API_KEY=****
 > Crossplatform
 
 ```bash
+npx -y rimraf backup
 npm run appwrite:backup
 ```
 
@@ -98,5 +104,23 @@ npm run appwrite:start:windows
 > Linux
 
 ```bash
+npx -y open-cli http://localhost:8080/
 npm run appwrite:start
 ```
+
+ - Authorize CLI in Docker AppWrite instance
+
+```bash
+npx -y open-cli http://localhost:8080/
+appwrite client --selfSigned true --endpoint http://localhost:8080/v1
+appwrite login
+```
+
+## File upload speed
+
+Looks like AppWrite file endpoint is limited `to 60 requests in every 1 minutes per IP address`. So [I added a delay](./scripts/restore.mjs), you can change it If you need to
+
+```javascript
+const DOCUMENT_WRITE_DELAY = 1_000;
+const FILE_WRITE_DELAY = 1_000;
+```

docker/.env

@@ -0,0 +1,94 @@
+_APP_ENV=production
+_APP_LOCALE=en
+_APP_OPTIONS_ABUSE=enabled
+_APP_OPTIONS_FORCE_HTTPS=disabled
+_APP_OPENSSL_KEY_V1=learning-key
+_APP_DOMAIN=localhost
+_APP_DOMAIN_TARGET=localhost
+_APP_CONSOLE_WHITELIST_ROOT=enabled
+_APP_CONSOLE_WHITELIST_EMAILS=
+_APP_CONSOLE_WHITELIST_IPS=
+_APP_SETUP=1-click-gitpod
+_APP_SYSTEM_EMAIL_NAME=Appwrite
+_APP_SYSTEM_EMAIL_ADDRESS=[email protected]
+_APP_SYSTEM_RESPONSE_FORMAT=
+_APP_SYSTEM_SECURITY_EMAIL_ADDRESS=[email protected]
+_APP_USAGE_STATS=enabled
+_APP_LOGGING_PROVIDER=
+_APP_LOGGING_CONFIG=
+_APP_USAGE_AGGREGATION_INTERVAL=30
+_APP_USAGE_TIMESERIES_INTERVAL=30
+_APP_USAGE_DATABASE_INTERVAL=900
+_APP_WORKER_PER_CORE=6
+_APP_REDIS_HOST=redis
+_APP_REDIS_PORT=6379
+_APP_REDIS_USER=
+_APP_REDIS_PASS=
+_APP_DB_HOST=mariadb
+_APP_DB_PORT=3306
+_APP_DB_SCHEMA=appwrite
+_APP_DB_USER=user
+_APP_DB_PASS=password
+_APP_DB_ROOT_PASS=rootsecretpassword
+_APP_INFLUXDB_HOST=influxdb
+_APP_INFLUXDB_PORT=8086
+_APP_STATSD_HOST=telegraf
+_APP_STATSD_PORT=8125
+_APP_SMTP_HOST=
+_APP_SMTP_PORT=
+_APP_SMTP_SECURE=
+_APP_SMTP_USERNAME=
+_APP_SMTP_PASSWORD=
+_APP_SMS_PROVIDER=
+_APP_SMS_FROM=
+_APP_STORAGE_LIMIT=10000000
+_APP_STORAGE_PREVIEW_LIMIT=20000000
+_APP_STORAGE_ANTIVIRUS=disabled
+_APP_STORAGE_ANTIVIRUS_HOST=clamav
+_APP_STORAGE_ANTIVIRUS_PORT=3310
+_APP_STORAGE_DEVICE=Local
+_APP_STORAGE_S3_ACCESS_KEY=
+_APP_STORAGE_S3_SECRET=
+_APP_STORAGE_S3_REGION=us-east-1
+_APP_STORAGE_S3_BUCKET=
+_APP_STORAGE_DO_SPACES_ACCESS_KEY=
+_APP_STORAGE_DO_SPACES_SECRET=
+_APP_STORAGE_DO_SPACES_REGION=us-east-1
+_APP_STORAGE_DO_SPACES_BUCKET=
+_APP_STORAGE_BACKBLAZE_ACCESS_KEY=
+_APP_STORAGE_BACKBLAZE_SECRET=
+_APP_STORAGE_BACKBLAZE_REGION=us-west-004
+_APP_STORAGE_BACKBLAZE_BUCKET=
+_APP_STORAGE_LINODE_ACCESS_KEY=
+_APP_STORAGE_LINODE_SECRET=
+_APP_STORAGE_LINODE_REGION=eu-central-1
+_APP_STORAGE_LINODE_BUCKET=
+_APP_STORAGE_WASABI_ACCESS_KEY=
+_APP_STORAGE_WASABI_SECRET=
+_APP_STORAGE_WASABI_REGION=eu-central-1
+_APP_STORAGE_WASABI_BUCKET=
+_APP_FUNCTIONS_SIZE_LIMIT=30000000
+_APP_FUNCTIONS_TIMEOUT=900
+_APP_FUNCTIONS_BUILD_TIMEOUT=900
+_APP_FUNCTIONS_CONTAINERS=10
+_APP_FUNCTIONS_CPUS=
+_APP_FUNCTIONS_MEMORY=256
+_APP_FUNCTIONS_MEMORY_SWAP=256
+_APP_FUNCTIONS_RUNTIMES=node-16.0,php-8.0,python-3.9,ruby-3.0,java-16.0
+_APP_EXECUTOR_SECRET=your-secret-key
+_APP_EXECUTOR_HOST=http://appwrite-executor/v1
+_APP_EXECUTOR_RUNTIME_NETWORK=integration-for-gitpod_runtimes
+_APP_FUNCTIONS_INACTIVE_THRESHOLD=60
+DOCKERHUB_PULL_USERNAME=
+DOCKERHUB_PULL_PASSWORD=
+DOCKERHUB_PULL_EMAIL=
+OPEN_RUNTIMES_NETWORK=integration-for-gitpod_runtimes
+_APP_MAINTENANCE_INTERVAL=86400
+_APP_MAINTENANCE_RETENTION_CACHE=2592000
+_APP_MAINTENANCE_RETENTION_EXECUTION=1209600
+_APP_MAINTENANCE_RETENTION_AUDIT=1209600
+_APP_MAINTENANCE_RETENTION_ABUSE=86400
+_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
+_APP_GRAPHQL_MAX_BATCH_SIZE=10
+_APP_GRAPHQL_MAX_COMPLEXITY=250
+_APP_GRAPHQL_MAX_DEPTH=3

docker/.gitignore

@@ -0,0 +1 @@
+!.env

package.json

@@ -9,8 +9,8 @@
     "appwrite:push": "sh ./tools/appwrite_push.sh",
     "appwrite:fetch:windows": ".\\tools\\appwrite_fetch.bat",
     "appwrite:push:windows": ".\\tools\\appwrite_push.bat",
-    "appwrite:start": "sh ./docker/run.sh",
-    "appwrite:start:windows": ".\\docker\\run.bat"
+    "appwrite:start": "cd docker && sh ./run.sh",
+    "appwrite:start:windows": "cd docker & run.bat"
   },
   "keywords": [],
   "author": "",

scripts/backup.mjs

@@ -19,13 +19,17 @@ const FILES_PAGE_SIZE = 100;
 const TOTAL_DOCUMENTS_LIMIT = 100_000;
 const DOCUMENTS_PAGE_SIZE = 100;
 
+const FILE_DOWNLOAD_DELAY = 1_000;
+
 const client = new sdk.Client();
 
-client
-  .setEndpoint(process.env.APPWRITE_ENDPOINT)
-  .setProject(process.env.APPWRITE_PROJECT_ID)
-  .setKey(process.env.APPWRITE_API_KEY)
-  .setSelfSigned();
+client.setEndpoint(process.env.APPWRITE_ENDPOINT)
+client.setProject(process.env.APPWRITE_PROJECT_ID)
+client.setKey(process.env.APPWRITE_API_KEY)
+
+if (process.env.APPWRITE_SELF_SIGNED) {
+  client.setSelfSigned();
+}
 
 const databases = new sdk.Databases(client);
 const storage = new sdk.Storage(client);
@@ -74,32 +78,16 @@ const listDocuments = async (databaseId, collectionId) => {
   return result;
 };
 
+const sleep = (timeout = 1_000) => new Promise((res) => {
+  setTimeout(() => {
+    res();
+  }, timeout);
+});
+
 await fs.mkdir("backup/buckets", { recursive: true });
 await fs.mkdir("backup/databases", { recursive: true });
 
-{
-  const { buckets, total } = await storage.listBuckets();
-
-  console.log(`Found ${total} buckets!`);
-
-  for (const bucket of buckets) {
-    console.log(`Getting list of files in ${bucket.$id}`);
-    const files = await listFiles(bucket.$id);
-    console.log(`Making backup of ${bucket.$id}`);
-    await fs.mkdir(`backup/buckets/${bucket.$id}`, { recursive: true });
-    for (const file of files) {
-      const metadata = await storage.getFile(bucket.$id, file.$id);
-      const extension = path.extname(metadata.name);
-      console.log(`Writing ${file.$id}${extension} from ${bucket.$id}`);
-      const buffer = await storage.getFileView(bucket.$id, file.$id);
-      await fs.writeFile(
-        `backup/buckets/${bucket.$id}/${file.$id}${extension}`,
-        buffer
-      );
-    }
-  }
-}
-
+// Backup documents
 {
   const { databases: databasesList, total } = await databases.list();
   console.log(`Found ${total} databases!`);
@@ -133,6 +121,29 @@ await fs.mkdir("backup/databases", { recursive: true });
   }
 }
 
+// Backup files
+{
+  const { buckets, total } = await storage.listBuckets();
+  console.log(`Found ${total} buckets!`);
+  for (const bucket of buckets) {
+    console.log(`Getting list of files in ${bucket.$id}`);
+    const files = await listFiles(bucket.$id);
+    console.log(`Making backup of ${bucket.$id}`);
+    await fs.mkdir(`backup/buckets/${bucket.$id}`, { recursive: true });
+    for (const file of files) {
+      const metadata = await storage.getFile(bucket.$id, file.$id);
+      const extension = path.extname(metadata.name);
+      console.log(`Writing ${file.$id}${extension} from ${bucket.$id}`);
+      const buffer = await storage.getFileView(bucket.$id, file.$id);
+      await fs.writeFile(
+        `backup/buckets/${bucket.$id}/${file.$id}${extension}`,
+        buffer
+      );
+      await sleep(FILE_DOWNLOAD_DELAY);
+    }
+  }
+}
+
 {
   console.log("Checking backup consistence...");
   const databaseFiles = await glob(`backup/databases/*/*/*.json`);