2.6.0-beta-1, 0.17.6-6, embedded file loader batching

manifest-beta.json

@@ -1,7 +1,7 @@
 {
 	"id": "obsidian-excalidraw-plugin",
 	"name": "Excalidraw",
-	"version": "2.5.3-beta-2",
+	"version": "2.6.0-beta-1",
 	"minAppVersion": "1.1.6",
 	"description": "An Obsidian plugin to edit and view Excalidraw drawings",
 	"author": "Zsolt Viczian",

package.json

@@ -19,7 +19,7 @@
   "license": "MIT",
   "dependencies": {
     "@popperjs/core": "^2.11.8",
-    "@zsviczian/excalidraw": "0.17.6-3",
+    "@zsviczian/excalidraw": "0.17.6-6",
     "chroma-js": "^2.4.2",
     "clsx": "^2.0.0",
     "@zsviczian/colormaster": "^1.2.2",
@@ -34,7 +34,8 @@
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "roughjs": "^4.5.2",
-    "woff2sfnt-sfnt2woff": "^1.0.0"
+    "woff2sfnt-sfnt2woff": "^1.0.0",
+    "es6-promise-pool": "2.5.0"
   },
   "devDependencies": {
     "@babel/core": "^7.22.9",

src/EmbeddedFileLoader.ts

@@ -36,6 +36,8 @@ import {
   isMaskFile,
   getEmbeddedFilenameParts,
   cropCanvas,
+  promiseTry,
+  PromisePool,
 } from "./utils/Utils";
 import { ValueOf } from "./types/types";
 import { getMermaidImageElements, getMermaidText, shouldRenderMermaid } from "./utils/MermaidUtils";
@@ -606,126 +608,169 @@ export class EmbeddedFilesLoader {
       this.isDark = excalidrawData?.scene?.appState?.theme === "dark";
     }
     let entry: IteratorResult<[FileId, EmbeddedFile]>;
-    const files: FileData[] = [];
-    while (!this.terminate && !(entry = entries.next()).done) {
-      if(fileIDWhiteList && !fileIDWhiteList.has(entry.value[0])) continue;
-      const embeddedFile: EmbeddedFile = entry.value[1];
-      if (!embeddedFile.isLoaded(this.isDark)) {
-        //debug({where:"EmbeddedFileLoader.loadSceneFiles",uid:this.uid,status:"embedded Files are not loaded"});
-        const data = await this._getObsidianImage(embeddedFile, depth);
-        if (data) {
-          const fileData: FileData = {
-            mimeType: data.mimeType,
-            id: entry.value[0],
-            dataURL: data.dataURL,
-            created: data.created,
-            size: data.size,
-            hasSVGwithBitmap: data.hasSVGwithBitmap,
-            shouldScale: embeddedFile.shouldScale()
-          };
-          try  {
-            addFiles([fileData], this.isDark, false);
-          }
-          catch(e) {
-            errorlog({ where: "EmbeddedFileLoader.loadSceneFiles", error: e });
-          }
-          //files.push(fileData);
-        }
-      } else if (embeddedFile.isSVGwithBitmap && (depth !== 0 || isThemeChange)) {
-        //this will reload the image in light/dark mode when switching themes
-        const fileData = {
-          mimeType: embeddedFile.mimeType,
-          id: entry.value[0],
-          dataURL: embeddedFile.getImage(this.isDark) as DataURL,
-          created: embeddedFile.mtime,
-          size: embeddedFile.size,
-          hasSVGwithBitmap: embeddedFile.isSVGwithBitmap,
-          shouldScale: embeddedFile.shouldScale()
-        };
-        //files.push(fileData);
-        try  {
-          addFiles([fileData], this.isDark, false);
-        }
-        catch(e) {
-          errorlog({ where: "EmbeddedFileLoader.loadSceneFiles", error: e });
-        }
-      }
-    }
+    const files: FileData[][] = [];
+    files.push([]);
+    let batch = 0;
 
-    let equation;
-    const equations = excalidrawData.getEquationEntries();
-    while (!this.terminate && !(equation = equations.next()).done) {
-      if(fileIDWhiteList && !fileIDWhiteList.has(equation.value[0])) continue;
-      if (!excalidrawData.getEquation(equation.value[0]).isLoaded) {
-        const latex = equation.value[1].latex;
-        const data = await tex2dataURL(latex);
-        if (data) {
-          const fileData = {
-            mimeType: data.mimeType,
-            id: equation.value[0],
-            dataURL: data.dataURL,
-            created: data.created,
-            size: data.size,
-            hasSVGwithBitmap: false,
-            shouldScale: true
-          };
-          files.push(fileData);
-        }
-      }
-    }
-
-    if(shouldRenderMermaid()) {
-      const mermaidElements = getMermaidImageElements(excalidrawData.scene.elements);
-      for(const element of mermaidElements) {
-        if(this.terminate) {
-          continue;
-        }
-        const data = getMermaidText(element);
-        const result = await mermaidToExcalidraw(data, {fontSize: 20}, true);
-        if(!result) {
-          continue;
-        }
-        if(result?.files) {
-          for (const key in result.files) {
+    function* loadIterator():Generator<Promise<void>> {
+      while (!(entry = entries.next()).done) {
+        if(fileIDWhiteList && !fileIDWhiteList.has(entry.value[0])) continue;
+        const embeddedFile: EmbeddedFile = entry.value[1];
+        const id = entry.value[0];
+        yield promiseTry(async () => {
+          if(this.terminate) {
+            return;
+          }
+          if (!embeddedFile.isLoaded(this.isDark)) {
+            //debug({where:"EmbeddedFileLoader.loadSceneFiles",uid:this.uid,status:"embedded Files are not loaded"});
+            const data = await this._getObsidianImage(embeddedFile, depth);
+            if (data) {
+              const fileData: FileData = {
+                mimeType: data.mimeType,
+                id: id,
+                dataURL: data.dataURL,
+                created: data.created,
+                size: data.size,
+                hasSVGwithBitmap: data.hasSVGwithBitmap,
+                shouldScale: embeddedFile.shouldScale()
+              };
+              files[batch].push(fileData);
+/*              try  {
+                addFiles([fileData], this.isDark, false);
+              }
+              catch(e) {
+                errorlog({ where: "EmbeddedFileLoader.loadSceneFiles", error: e });
+              }*/
+            }
+          } else if (embeddedFile.isSVGwithBitmap && (depth !== 0 || isThemeChange)) {
+            //this will reload the image in light/dark mode when switching themes
             const fileData = {
-              ...result.files[key],
-              id: element.fileId,
-              created: Date.now(),
-              hasSVGwithBitmap: false,
-              shouldScale: true,
-              size: await getImageSize(result.files[key].dataURL),
+              mimeType: embeddedFile.mimeType,
+              id: id,
+              dataURL: embeddedFile.getImage(this.isDark) as DataURL,
+              created: embeddedFile.mtime,
+              size: embeddedFile.size,
+              hasSVGwithBitmap: embeddedFile.isSVGwithBitmap,
+              shouldScale: embeddedFile.shouldScale()
             };
-            files.push(fileData);
+            files[batch].push(fileData);
+/*            try  {
+              addFiles([fileData], this.isDark, false);
+            }
+            catch(e) {
+              errorlog({ where: "EmbeddedFileLoader.loadSceneFiles", error: e });
+            }*/
           }
-          continue;
-        }
-        if(result?.elements) {
-          //handle case that mermaidToExcalidraw has implemented this type of diagram in the mean time
-          const res = await this.getExcalidrawSVG({
-            isDark: this.isDark,
-            file: null,
-            depth,
-            inFile: null,
-            hasSVGwithBitmap: false,
-            elements: result.elements
-          });
-          if(res?.dataURL) {
-            const size = await getImageSize(res.dataURL);
-            const fileData:FileData = {
-              mimeType: "image/svg+xml",
-              id: element.fileId,
-              dataURL: res.dataURL,
-              created: Date.now(),
-              hasSVGwithBitmap: res.hasSVGwithBitmap,
-              size,
-              shouldScale: true,
-            };
-            files.push(fileData);
-          }
-          continue;
-        }  
+        });
       }
-    };
+
+      let equationItem;
+      const equations = excalidrawData.getEquationEntries();
+      while (!(equationItem = equations.next()).done) {
+        if(fileIDWhiteList && !fileIDWhiteList.has(equationItem.value[0])) continue;
+        const equation = equationItem.value[1];
+        const id = equationItem.value[0];
+        yield promiseTry(async () => {
+          if (this.terminate) {
+            return;
+          }
+          if (!excalidrawData.getEquation(id).isLoaded) {
+            const latex = equation.latex;
+            const data = await tex2dataURL(latex);
+            if (data) {
+              const fileData = {
+                mimeType: data.mimeType,
+                id: id,
+                dataURL: data.dataURL,
+                created: data.created,
+                size: data.size,
+                hasSVGwithBitmap: false,
+                shouldScale: true
+              };
+              files[batch].push(fileData);
+            }
+          }
+        });
+      }
+
+      if(shouldRenderMermaid()) {
+        const mermaidElements = getMermaidImageElements(excalidrawData.scene.elements);
+        for(const element of mermaidElements) {
+          yield promiseTry(async () => {
+            if(this.terminate) {
+              return;
+            }
+            const data = getMermaidText(element);
+            const result = await mermaidToExcalidraw(data, {fontSize: 20}, true);
+            if(!result) {
+              return;
+            }
+            if(result?.files) {
+              for (const key in result.files) {
+                const fileData = {
+                  ...result.files[key],
+                  id: element.fileId,
+                  created: Date.now(),
+                  hasSVGwithBitmap: false,
+                  shouldScale: true,
+                  size: await getImageSize(result.files[key].dataURL),
+                };
+                files[batch].push(fileData);
+              }
+              return;
+            }
+            if(result?.elements) {
+              //handle case that mermaidToExcalidraw has implemented this type of diagram in the mean time
+              if (this.terminate) {
+                return;
+              }
+              const res = await this.getExcalidrawSVG({
+                isDark: this.isDark,
+                file: null,
+                depth,
+                inFile: null,
+                hasSVGwithBitmap: false,
+                elements: result.elements
+              });
+              if(res?.dataURL) {
+                const size = await getImageSize(res.dataURL);
+                const fileData:FileData = {
+                  mimeType: "image/svg+xml",
+                  id: element.fileId,
+                  dataURL: res.dataURL,
+                  created: Date.now(),
+                  hasSVGwithBitmap: res.hasSVGwithBitmap,
+                  size,
+                  shouldScale: true,
+                };
+                files[batch].push(fileData);
+              }
+              return;
+            }
+          });
+        }
+      };
+    }
+
+    const addFilesTimer = setInterval(() => {
+      if(files[batch].length === 0) {
+        return;
+      }
+      try  {
+        addFiles(files[batch], this.isDark, false);
+      }
+      catch(e) {
+        errorlog({ where: "EmbeddedFileLoader.loadSceneFiles", error: e });
+      }
+      files.push([]);
+      batch++;
+    }, 1200);
+
+    const iterator = loadIterator.bind(this)();
+    const concurency = 5;
+    await new PromisePool(iterator, concurency).all();
+    
+    clearInterval(addFilesTimer);
 
     this.emptyPDFDocsMap();
     if (this.terminate) {
@@ -734,7 +779,7 @@ export class EmbeddedFilesLoader {
     //debug({where:"EmbeddedFileLoader.loadSceneFiles",uid:this.uid,status:"add Files"});
     try {
       //in try block because by the time files are loaded the user may have closed the view
-      addFiles(files, this.isDark, true);
+      addFiles(files[batch], this.isDark, true);
     } catch (e) {
       errorlog({ where: "EmbeddedFileLoader.loadSceneFiles", error: e });
     }

src/ExcalidrawLib.d.ts

@@ -189,6 +189,5 @@ declare namespace ExcalidrawLib {
   ): string;
   function safelyParseJSON (json: string): Record<string, any> | null;
   function loadSceneFonts(elements: NonDeletedExcalidrawElement[]): Promise<void>;
-  function initializeObsidianUtils(obsidianPlugin: ExcalidrawPlugin): void;
 }
 

src/constants/constants.ts

@@ -104,7 +104,6 @@ export const {
   refreshTextDimensions,
   getCSSFontDefinition,
   loadSceneFonts,
-  initializeObsidianUtils,
 } = excalidrawLib;
 
 export const FONTS_STYLE_ID = "excalidraw-custom-fonts";

src/main.ts

@@ -45,7 +45,6 @@ import {
   DEVICE,
   sceneCoordsToViewportCoords,
   FONTS_STYLE_ID,
-  initializeObsidianUtils,
 } from "./constants/constants";
 import ExcalidrawView, { TextMode, getTextMode } from "./ExcalidrawView";
 import {
@@ -205,7 +204,6 @@ export default class ExcalidrawPlugin extends Plugin {
     this.equationsMaster = new Map<FileId, string>();
     this.mermaidsMaster = new Map<FileId, string>();
     setExcalidrawPlugin(this);
-    initializeObsidianUtils(this);
     /*if((process.env.NODE_ENV === 'development')) {
       this.slob = new Array(200 * 1024 * 1024 + 1).join('A'); // Create a 200MB blob
     }*/

src/utils/Utils.ts

@@ -29,6 +29,7 @@ import { updateElementLinksToObsidianLinks } from "src/ExcalidrawAutomate";
 import { CropImage } from "./CropImage";
 import opentype from 'opentype.js';
 import { runCompressionWorker } from "src/workers/compression-worker";
+import Pool from "es6-promise-pool";
 
 declare const PLUGIN_VERSION:string;
 declare var LZString: any;
@@ -968,4 +969,63 @@ export function cropCanvas(
       0, 0, output.width, output.height
   );
   return dstCanvas;
+}
+
+// Promise.try, adapted from https://github.com/sindresorhus/p-try
+export async function promiseTry <TValue, TArgs extends unknown[]>(
+  fn: (...args: TArgs) => PromiseLike<TValue> | TValue,
+  ...args: TArgs
+): Promise<TValue> {
+  return new Promise((resolve) => {
+    resolve(fn(...args));
+  });
+};
+
+// extending the missing types
+// relying on the [Index, T] to keep a correct order
+type TPromisePool<T, Index = number> = Pool<[Index, T][]> & {
+  addEventListener: (
+    type: "fulfilled",
+    listener: (event: { data: { result: [Index, T] } }) => void,
+  ) => (event: { data: { result: [Index, T] } }) => void;
+  removeEventListener: (
+    type: "fulfilled",
+    listener: (event: { data: { result: [Index, T] } }) => void,
+  ) => void;
+};
+
+export class PromisePool<T> {
+  private readonly pool: TPromisePool<T>;
+  private readonly entries: Record<number, T> = {};
+
+  constructor(
+    source: IterableIterator<Promise<void | readonly [number, T]>>,
+    concurrency: number,
+  ) {
+    this.pool = new Pool(
+      source as unknown as () => void | PromiseLike<[number, T][]>,
+      concurrency,
+    ) as TPromisePool<T>;
+  }
+
+  public all() {
+    const listener = (event: { data: { result: void | [number, T] } }) => {
+      if (event.data.result) {
+        // by default pool does not return the results, so we are gathering them manually
+        // with the correct call order (represented by the index in the tuple)
+        const [index, value] = event.data.result;
+        this.entries[index] = value;
+      }
+    };
+
+    this.pool.addEventListener("fulfilled", listener);
+
+    return this.pool.start().then(() => {
+      setTimeout(() => {
+        this.pool.removeEventListener("fulfilled", listener);
+      });
+
+      return Object.values(this.entries);
+    });
+  }
 }