feat(front): parallelize fragment file streaming

ThatOpen · Oct 13, 2024 · c80748f · c80748f
1 parent 5d686f2
commit c80748f
Show file tree

Hide file tree

Showing 5 changed files with 165 additions and 134 deletions.
diff --git a/packages/core/package.json b/packages/core/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@thatopen/components",
   "description": "Collection of core functionalities to author BIM apps.",
-  "version": "2.3.4",
+  "version": "2.4.0-alpha.1",
   "author": "That Open Company",
   "contributors": [
     "Antonio Gonzalez Viegas (https://github.com/agviegas)",
@@ -37,7 +37,7 @@
     "access": "public"
   },
   "devDependencies": {
-    "@thatopen/fragments": "~2.3.0",
+    "@thatopen/fragments": ">=2.4.0-alpha.1",
     "@thatopen/ui": "~2.3.0",
     "@types/three": "0.160.0",
     "stats.js": "^0.17.0",
@@ -51,8 +51,8 @@
     "three-mesh-bvh": "0.7.0"
   },
   "peerDependencies": {
-    "@thatopen/fragments": "~2.3.0",
+    "@thatopen/fragments": ">=2.4.0-alpha.1",
     "three": "^0.160.1",
     "web-ifc": "0.0.59"
   }
-}
+}
diff --git a/packages/core/src/core/Components/index.ts b/packages/core/src/core/Components/index.ts
@@ -14,7 +14,7 @@ export class Components implements Disposable {
   /**
    * The version of the @thatopen/components library.
    */
-  static readonly release = "2.3.4";
+  static readonly release = "2.4.0-alpha.1";
 
   /** {@link Disposable.onDisposed} */
   readonly onDisposed = new Event<void>();

diff --git a/packages/front/package.json b/packages/front/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@thatopen/components-front",
   "description": "Collection of frontend tools to author BIM apps.",
-  "version": "2.3.3",
+  "version": "2.4.0-alpha.1",
   "author": "That Open Company",
   "contributors": [
     "Antonio Gonzalez Viegas (https://github.com/agviegas)",
@@ -33,12 +33,12 @@
     "publish-alpha": "npm publish --tag alpha"
   },
   "peerDependencies": {
-    "@thatopen/fragments": ">=2.3.0-alpha",
+    "@thatopen/fragments": ">=2.4.0-alpha.1",
     "three": "^0.160.1",
     "web-ifc": "0.0.59"
   },
   "devDependencies": {
-    "@thatopen/fragments": "~2.3.0",
+    "@thatopen/fragments": ">=2.4.0-alpha.1",
     "@thatopen/ui": "~2.3.0",
     "@thatopen/ui-obc": "~2.3.0",
     "@types/earcut": "^2.1.4",
@@ -47,11 +47,11 @@
     "web-ifc": "0.0.59"
   },
   "dependencies": {
-    "@thatopen/components": ">=2.3.0",
+    "@thatopen/components": ">=2.4.0-alpha.1",
     "camera-controls": "2.7.3",
     "dexie": "^4.0.4",
     "earcut": "^2.2.4",
     "n8ao": "1.5.1",
     "postprocessing": "6.34.2"
   }
-}
+}
diff --git a/packages/front/src/fragments/IfcStreamer/index.ts b/packages/front/src/fragments/IfcStreamer/index.ts
@@ -494,126 +494,31 @@ export class IfcStreamer extends OBC.Component implements OBC.Disposable {
         }
       }
 
-      const sortedFiles = Array.from(files).sort((a, b) => b[1] - a[1]);
-
-      for (const [fileName] of sortedFiles) {
-        // If this file is still in the ram, get it
-
-        if (!this._ramCache.has(fileName)) {
-          let bytes = new Uint8Array();
-
-          // If this file is in the local cache, get it
-          if (this.useCache) {
-            // Add or update this file to clean it up from indexedDB automatically later
-
-            const found = await this.fileDB.get(fileName);
-
-            if (found) {
-              const arrayBuffer = await found.arrayBuffer();
-              bytes = new Uint8Array(arrayBuffer);
-            } else {
-              const fetched = await this.fetch(fileName);
-              const buffer = await fetched.arrayBuffer();
-              bytes = new Uint8Array(buffer);
-              await this.fileDB.add(fileName, bytes);
-            }
-          } else {
-            const fetched = await this.fetch(fileName);
-            const buffer = await fetched.arrayBuffer();
-            bytes = new Uint8Array(buffer);
-          }
-
-          const data = this.serializer.import(bytes);
-          this._ramCache.set(fileName, { data, time: performance.now() });
-        }
-
-        const result = this._ramCache.get(fileName);
-        if (!result) {
-          continue;
-        }
-
-        result.time = performance.now();
-
-        const loaded: FRAG.Fragment[] = [];
-
-        if (result) {
-          for (const [geometryID, { position, index, normal }] of result.data) {
-            if (this._isDisposing) {
-              return;
-            }
-
-            if (this.cancel) {
-              this.cancelLoading(cancelled);
-              return;
-            }
-
-            // This fragment can be cancelled, it will be loaded
-            cancelled[modelID].delete(geometryID);
-
-            if (!allIDs.has(geometryID)) continue;
-
-            if (
-              !this._geometryInstances[modelID] ||
-              !this._geometryInstances[modelID].has(geometryID)
-            ) {
-              continue;
-            }
-
-            const geoms = this._geometryInstances[modelID];
-            const instances = geoms.get(geometryID);
-
-            if (!instances) {
-              throw new Error("Instances not found!");
-            }
-
-            const geom = new THREE.BufferGeometry();
-
-            const posAttr = new THREE.BufferAttribute(position, 3);
-            const norAttr = new THREE.BufferAttribute(normal, 3);
-
-            geom.setAttribute("position", posAttr);
-            geom.setAttribute("normal", norAttr);
-
-            geom.setIndex(Array.from(index));
-
-            // Separating opaque and transparent items is neccesary for Three.js
-
-            const transp: StreamedInstance[] = [];
-            const opaque: StreamedInstance[] = [];
-            for (const instance of instances) {
-              if (instance.color[3] === 1) {
-                opaque.push(instance);
-              } else {
-                transp.push(instance);
-              }
-            }
-
-            this.newFragment(
-              group,
-              geometryID,
-              geom,
-              transp,
-              true,
-              loaded,
-              visible,
-            );
-
-            this.newFragment(
-              group,
-              geometryID,
-              geom,
-              opaque,
-              false,
-              loaded,
-              visible,
-            );
+      // Give more priority to the files that are already cached
+      if (this.useCache) {
+        const cachedPriority = 99999;
+        const entries = files.entries();
+        for (const [name, value] of entries) {
+          if (this.fileDB.isCached(name)) {
+            files.set(name, value + cachedPriority);
           }
         }
+      }
 
-        if (loaded.length && !this._isDisposing) {
-          this.onFragmentsLoaded.trigger(loaded);
-        }
+      const sortedFiles = Array.from(files).sort((a, b) => b[1] - a[1]);
+      const loadProcesses: Promise<void>[] = [];
+      for (const [fileName] of sortedFiles) {
+        const loadProcess = this.loadFragmentFile(
+          modelID,
+          group,
+          visible,
+          fileName,
+          allIDs,
+          cancelled,
+        );
+        loadProcesses.push(loadProcess);
       }
+      await Promise.all(loadProcesses);
 
       const expiredIDs = new Set<string>();
       const now = performance.now();
@@ -815,4 +720,130 @@ export class IfcStreamer extends OBC.Component implements OBC.Disposable {
     this.cancel = false;
     this.culler.cancel(items);
   }
+
+  private async loadFragmentFile(
+    modelID: string,
+    group: FRAG.FragmentsGroup,
+    visible: boolean,
+    fileName: string,
+    allIDs: Set<number>,
+    cancelled: { [modelID: string]: Set<number> },
+  ) {
+    // If this file is still in the ram, get it
+
+    if (!this._ramCache.has(fileName)) {
+      let bytes = new Uint8Array();
+
+      // If this file is in the local cache, get it
+      if (this.useCache) {
+        // Add or update this file to clean it up from indexedDB automatically later
+
+        const found = await this.fileDB.get(fileName);
+
+        if (found) {
+          const arrayBuffer = await found.arrayBuffer();
+          bytes = new Uint8Array(arrayBuffer);
+        } else {
+          const fetched = await this.fetch(fileName);
+          const buffer = await fetched.arrayBuffer();
+          bytes = new Uint8Array(buffer);
+          await this.fileDB.add(fileName, bytes);
+        }
+      } else {
+        const fetched = await this.fetch(fileName);
+        const buffer = await fetched.arrayBuffer();
+        bytes = new Uint8Array(buffer);
+      }
+
+      const data = this.serializer.import(bytes);
+      this._ramCache.set(fileName, { data, time: performance.now() });
+    }
+
+    const result = this._ramCache.get(fileName);
+    if (!result) {
+      return;
+    }
+
+    result.time = performance.now();
+
+    const loaded: FRAG.Fragment[] = [];
+
+    if (result) {
+      for (const [geometryID, { position, index, normal }] of result.data) {
+        if (this._isDisposing) {
+          return;
+        }
+
+        if (this.cancel) {
+          this.cancelLoading(cancelled);
+          return;
+        }
+
+        // This fragment can be cancelled, it will be loaded
+        cancelled[modelID].delete(geometryID);
+
+        if (!allIDs.has(geometryID)) continue;
+
+        if (
+          !this._geometryInstances[modelID] ||
+          !this._geometryInstances[modelID].has(geometryID)
+        ) {
+          continue;
+        }
+
+        const geoms = this._geometryInstances[modelID];
+        const instances = geoms.get(geometryID);
+
+        if (!instances) {
+          throw new Error("Instances not found!");
+        }
+
+        const geom = new THREE.BufferGeometry();
+
+        const posAttr = new THREE.BufferAttribute(position, 3);
+        const norAttr = new THREE.BufferAttribute(normal, 3);
+
+        geom.setAttribute("position", posAttr);
+        geom.setAttribute("normal", norAttr);
+
+        geom.setIndex(Array.from(index));
+
+        // Separating opaque and transparent items is neccesary for Three.js
+
+        const transp: StreamedInstance[] = [];
+        const opaque: StreamedInstance[] = [];
+        for (const instance of instances) {
+          if (instance.color[3] === 1) {
+            opaque.push(instance);
+          } else {
+            transp.push(instance);
+          }
+        }
+
+        this.newFragment(
+          group,
+          geometryID,
+          geom,
+          transp,
+          true,
+          loaded,
+          visible,
+        );
+
+        this.newFragment(
+          group,
+          geometryID,
+          geom,
+          opaque,
+          false,
+          loaded,
+          visible,
+        );
+      }
+    }
+
+    if (loaded.length && !this._isDisposing) {
+      this.onFragmentsLoaded.trigger(loaded);
+    }
+  }
 }