v0.0.19

Author: kaiiiz

.prettierrc

@@ -0,0 +1,3 @@
+{
+	"printWidth": 160
+}

README.md

@@ -29,7 +29,7 @@ Use `Raindrop Highlights: Open link in Raindrop` command to open the correspondi
 
 Use `Raindrop Highlights: Manage collections to be synced` command to quickly open the collection management modal.
 
-**NOTE**: Do not touch the front matter properties: `raindrop_id` and `raindrop_last_update`. These properties are used to identify the existing article to prevent file and highlights duplication.
+**NOTE**: Do not touch the front matter properties: `raindrop_id` and `raindrop_highlights`. These properties are used to identify the existing article to prevent file and highlights duplication.
 
 ### API Token
 

esbuild.config.mjs

@@ -59,7 +59,7 @@ esbuild.build({
 		...builtins],
 	format: 'cjs',
 	watch: !prod,
-	target: 'es2016',
+	target: 'es2018',
 	logLevel: "info",
 	sourcemap: prod ? false : 'inline',
 	treeShaking: true,

manifest.json

@@ -1,7 +1,7 @@
 {
 	"id": "obsidian-raindrop-highlights",
 	"name": "Raindrop Highlights",
-	"version": "0.0.18",
+	"version": "0.0.19",
 	"minAppVersion": "0.14.0",
 	"description": "Sync your Raindrop.io highlights.",
 	"author": "kaiiiz",

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "obsidian-raindrop-highlights",
-	"version": "0.0.18",
+	"version": "0.0.19",
 	"description": "Sync your Raindrop.io highlights.",
 	"main": "main.js",
 	"scripts": {
@@ -14,24 +14,26 @@
 	"devDependencies": {
 		"@tsconfig/svelte": "3.0.0",
 		"@types/node": "16.11.6",
-		"@types/nunjucks": "3.2.1",
+		"@types/nunjucks": "3.2.4",
 		"@types/semver": "^7.5.0",
 		"@types/truncate-utf8-bytes": "^1.0.0",
 		"@typescript-eslint/eslint-plugin": "5.29.0",
 		"@typescript-eslint/parser": "5.29.0",
 		"builtin-modules": "3.3.0",
 		"esbuild": "0.14.47",
 		"esbuild-svelte": "0.7.1",
-		"obsidian": "^1.2.8",
+		"obsidian": "^1.4.11",
 		"svelte": "3.49.0",
 		"svelte-preprocess": "4.10.7",
 		"tslib": "2.4.0",
 		"typescript": "4.7.4"
 	},
 	"dependencies": {
 		"axios": "0.27.2",
-		"nunjucks": "3.2.3",
+		"axios-retry": "^3.8.0",
+		"nunjucks": "3.2.4",
 		"sanitize-filename": "1.6.3",
-		"semver": "^7.5.2"
+		"semver": "^7.5.2",
+		"ts-md5": "^1.3.1"
 	}
 }

src/api.ts

@@ -1,15 +1,37 @@
 import { Notice, type App } from "obsidian";
-import axios from "axios";
+import axios, { AxiosError } from "axios";
+import axiosRetry from "axios-retry";
 import type { RaindropBookmark, RaindropCollection, RaindropCollectionGroup, RaindropHighlight, RaindropUser } from "./types";
 import TokenManager from "./tokenManager";
+import { Md5 } from "ts-md5";
 
 const BASEURL = "https://api.raindrop.io/rest/v1";
 
 interface NestedRaindropCollection {
-	title: string,
-	parentId: number,
+	title: string;
+	parentId: number;
 }
 
+axiosRetry(axios, {
+	retries: 3,
+	retryCondition: (error: AxiosError) => {
+		if (error.response && error.response.status === 429) {
+			new Notice("Too many requests, will retry sync after 1 minute", 5);
+			console.warn(`Too many requests, will retry sync after 1 minute`);
+			return true;
+		} else {
+			console.error(`request error: ${error}`);
+		}
+		return false;
+	},
+	retryDelay: () => {
+		return 60000;
+	},
+	onRetry: (retryCount) => {
+		new Notice(`Retry sync ${retryCount}/3`);
+	},
+});
+
 export class RaindropAPI {
 	app: App;
 	tokenManager: TokenManager;
@@ -28,7 +50,7 @@ export class RaindropAPI {
 		const result = await axios.get(url, {
 			params: params,
 			headers: {
-				"Authorization": `Bearer ${token}`,
+				Authorization: `Bearer ${token}`,
 				"Content-Type": "application/json",
 			},
 		});
@@ -50,12 +72,12 @@ export class RaindropAPI {
 		const nestedCollectionPromise = this.get(`${BASEURL}/collections/childrens`, {});
 
 		const collections: RaindropCollection[] = [
-			{ id: -1, title: 'Unsorted' },
-			{ id: 0, title: 'All bookmarks' },
-			{ id: -99, title: 'Trash' },
+			{ id: -1, title: "Unsorted" },
+			{ id: 0, title: "All bookmarks" },
+			{ id: -99, title: "Trash" },
 		];
 
-		const collectionGroupMap: {[id: number]: string} = {};
+		const collectionGroupMap: { [id: number]: string } = {};
 		if (enableCollectionGroup) {
 			const res = await this.get(`${BASEURL}/user`, {});
 			const groups = this.parseGroups(res.user.groups);
@@ -66,11 +88,11 @@ export class RaindropAPI {
 			});
 		}
 
-		const rootCollectionMap: {[id: number]: string} = {};
+		const rootCollectionMap: { [id: number]: string } = {};
 		const rootCollections = await rootCollectionPromise;
 		rootCollections.items.forEach((collection: any) => {
-			const id = collection['_id'];
-			let title = collection['title'];
+			const id = collection["_id"];
+			let title = collection["title"];
 			if (enableCollectionGroup) {
 				title = `${collectionGroupMap[id]}/${title}`;
 			}
@@ -81,25 +103,25 @@ export class RaindropAPI {
 			});
 		});
 
-		const nestedCollectionMap: {[id: number]: NestedRaindropCollection} = {};
+		const nestedCollectionMap: { [id: number]: NestedRaindropCollection } = {};
 		const nestedCollections = await nestedCollectionPromise;
 		nestedCollections.items.forEach((collection: any) => {
-			const id = collection['_id'];
+			const id = collection["_id"];
 			nestedCollectionMap[id] = {
-				title: collection['title'],
-				parentId: collection['parent']['$id'],
+				title: collection["title"],
+				parentId: collection["parent"]["$id"],
 			};
 		});
 
 		nestedCollections.items.forEach((collection: any) => {
-			const id = collection['_id'];
-			let parentId = collection['parent']['$id'];
-			let title = collection['title'];
-			while (parentId && (parentId in nestedCollectionMap)) {
+			const id = collection["_id"];
+			let parentId = collection["parent"]["$id"];
+			let title = collection["title"];
+			while (parentId && parentId in nestedCollectionMap) {
 				title = `${nestedCollectionMap[parentId].title}/${title}`;
 				parentId = nestedCollectionMap[parentId].parentId;
 			}
-			if (parentId && (parentId in rootCollectionMap)) {
+			if (parentId && parentId in rootCollectionMap) {
 				title = `${rootCollectionMap[parentId]}/${title}`;
 			}
 			collections.push({
@@ -111,54 +133,59 @@ export class RaindropAPI {
 		return collections;
 	}
 
-	async getRaindropsAfter(collectionId: number, lastSync?: Date): Promise<RaindropBookmark[]> {
-		const notice = new Notice("Fetch Raindrops highlights", 0);
+	async *getRaindropsAfter(collectionId: number, showNotice: boolean, lastSync?: Date): AsyncGenerator<RaindropBookmark[]> {
+		let notice;
+		if (showNotice) {
+			notice = new Notice("Fetch Raindrops highlights", 0);
+		}
+
+		const pageSize = 50;
 		const res = await this.get(`${BASEURL}/raindrops/${collectionId}`, {
-			"page": 0,
-			"sort": "-lastUpdate"
+			page: 0,
+			perpage: pageSize,
+			sort: "-lastUpdate",
 		});
 		const raindropsCnt = res.count;
 		let bookmarks = this.parseRaindrops(res.items);
-		let remainPages = Math.ceil(raindropsCnt / 25) - 1;
-		const totalPages = Math.ceil(raindropsCnt / 25) - 1;
+		const totalPages = Math.ceil(raindropsCnt / pageSize);
+		let remainPages = totalPages - 1;
 		let page = 1;
 
-		const addNewPages = async (page: number) => {
+		const getPage = async (page: number) => {
 			const res = await this.get(`${BASEURL}/raindrops/${collectionId}`, {
-				"page": page,
-				"sort": "-lastUpdate"
+				page: page,
+				perpage: pageSize,
+				sort: "-lastUpdate",
 			});
-			bookmarks = bookmarks.concat(this.parseRaindrops(res.items));
-		}
+			return this.parseRaindrops(res.items);
+		};
 
-		if (bookmarks.length > 0) {
-			if (lastSync === undefined) { // sync all
+		if (lastSync === undefined) {
+			if (bookmarks.length > 0) {
+				yield bookmarks;
 				while (remainPages--) {
-					notice.setMessage(`Sync Raindrop pages: ${totalPages - remainPages}/${totalPages}`)
-					await addNewPages(page++);
-				}
-			} else { // sync article after lastSync
-				while (bookmarks[bookmarks.length - 1].lastUpdate.getTime() >= lastSync.getTime() && remainPages--) {
-					notice.setMessage(`Sync Raindrop pages: ${totalPages - remainPages}/${totalPages}`)
-					await addNewPages(page++);
+					notice?.setMessage(`Sync Raindrop pages: ${page + 1}/${totalPages}`);
+					yield await getPage(page++);
 				}
-				bookmarks = bookmarks.filter(bookmark => {
+			}
+		} else {
+			const filterLastUpdate = (bookmarks: RaindropBookmark[]) => {
+				return bookmarks.filter((bookmark) => {
 					return bookmark.lastUpdate.getTime() >= lastSync.getTime();
 				});
+			};
+			const filteredBookmark = filterLastUpdate(bookmarks);
+			if (filteredBookmark.length > 0) {
+				yield filteredBookmark;
+				while (bookmarks[bookmarks.length - 1].lastUpdate.getTime() >= lastSync.getTime() && remainPages--) {
+					notice?.setMessage(`Sync Raindrop pages: ${page + 1}/${totalPages}`);
+					let bookmarks = await getPage(page++);
+					yield filterLastUpdate(bookmarks);
+				}
 			}
 		}
 
-		// get real highlights (raindrop returns only 3 highlights in /raindrops/${collectionId} endpoint)
-		for (const [idx, bookmark] of bookmarks.entries()) {
-			notice.setMessage(`Sync Raindrop bookmarks: ${idx + 1}/${bookmarks.length}`)
-			if (bookmark.highlights.length == 3) {
-				const res = await this.get(`${BASEURL}/raindrop/${bookmark.id}`, {});
-				bookmark['highlights'] = this.parseHighlights(res.item.highlights);
-			}
-		}
-
-		notice.hide();
-		return bookmarks;
+		notice?.hide();
 	}
 
 	async getUser(): Promise<RaindropUser> {
@@ -173,14 +200,14 @@ export class RaindropAPI {
 		try {
 			result = await axios.get(`${BASEURL}/user`, {
 				headers: {
-					"Authorization": `Bearer ${token}`,
+					Authorization: `Bearer ${token}`,
 					"Content-Type": "application/json",
 				},
 			});
 			if (result.status !== 200) {
 				throw new Error("Invalid token");
 			}
-		} catch(e) {
+		} catch (e) {
 			throw new Error("Invalid token");
 		}
 
@@ -204,22 +231,22 @@ export class RaindropAPI {
 
 	private parseRaindrop(raindrop: any): RaindropBookmark {
 		const bookmark: RaindropBookmark = {
-			id: raindrop['_id'],
-			collectionId: raindrop['collectionId'],
-			title: raindrop['title'],
-			highlights: this.parseHighlights(raindrop['highlights']),
-			excerpt: raindrop['excerpt'],
-			note: raindrop['note'],
-			link: raindrop['link'],
-			lastUpdate: new Date(raindrop['lastUpdate']),
-			tags: raindrop['tags'],
-			cover: raindrop['cover'],
-			created: new Date(raindrop['created']),
-			type: raindrop['type'],
-			important: raindrop['important'],
+			id: raindrop["_id"],
+			collectionId: raindrop["collectionId"],
+			title: raindrop["title"],
+			highlights: this.parseHighlights(raindrop["highlights"]),
+			excerpt: raindrop["excerpt"],
+			note: raindrop["note"],
+			link: raindrop["link"],
+			lastUpdate: new Date(raindrop["lastUpdate"]),
+			tags: raindrop["tags"],
+			cover: raindrop["cover"],
+			created: new Date(raindrop["created"]),
+			type: raindrop["type"],
+			important: raindrop["important"],
 			creator: {
-				name: raindrop['creatorRef']['name'],
-				id: raindrop['creatorRef']['_id'],
+				name: raindrop["creatorRef"]["name"],
+				id: raindrop["creatorRef"]["_id"],
 			},
 		};
 		return bookmark;
@@ -228,12 +255,13 @@ export class RaindropAPI {
 	private parseHighlights(highlights: any): RaindropHighlight[] {
 		return highlights.map((hl: any) => {
 			const highlight: RaindropHighlight = {
-				id: hl['_id'],
-				color: hl['color'],
-				text: hl['text'],
-				lastUpdate: new Date(hl['lastUpdate']),
-				created: new Date(hl['created']),
-				note: hl['note'],
+				id: hl["_id"],
+				color: hl["color"],
+				text: hl["text"],
+				lastUpdate: new Date(hl["lastUpdate"]),
+				created: new Date(hl["created"]),
+				note: hl["note"],
+				signature: Md5.hashStr(`${hl["color"]},${hl["text"]},${hl["note"]}`),
 			};
 			return highlight;
 		});
@@ -242,8 +270,8 @@ export class RaindropAPI {
 	private parseGroups(groups: any): RaindropCollectionGroup[] {
 		return groups.map((g: any) => {
 			const group: RaindropCollectionGroup = {
-				title: g['title'],
-				collections: g['collections'],
+				title: g["title"],
+				collections: g["collections"],
 			};
 			return group;
 		});

src/constants.ts

@@ -1,7 +1,7 @@
-import DEFAULT_TEMPLATE from './assets/defaultTemplate.njk';
+import DEFAULT_TEMPLATE from "./assets/defaultTemplate.njk";
 import type { RaindropPluginSettings } from "./types";
 
-export const VERSION = '0.0.18';
+export const VERSION = "0.0.19";
 
 export const DEFAULT_SETTINGS: RaindropPluginSettings = {
 	version: VERSION,
@@ -11,11 +11,13 @@ export const DEFAULT_SETTINGS: RaindropPluginSettings = {
 	appendMode: true,
 	collectionsFolders: true,
 	onlyBookmarksWithHl: false,
-	highlightsFolder: '/',
+	highlightsFolder: "/",
 	collectionGroups: false,
+	autoSyncSuccessNotice: true,
 	syncCollections: {},
 	template: DEFAULT_TEMPLATE,
 	metadataTemplate: "",
 	filenameTemplate: "{{title}}",
 	autoSyncInterval: 0,
+	autoescape: true,
 };