Add multithreaded decompression of archives

ixray-team · Dec 4, 2024 · 3482404 · 3482404
1 parent 65a8b00
commit 3482404
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 10 deletions.
diff --git a/src/utils/xrCompress/xrDecompress.cpp b/src/utils/xrCompress/xrDecompress.cpp
@@ -1,7 +1,7 @@
 #include "StdAfx.h"
 #include "xrDecompress.h"
 
-const char xrDecompressor::s_Separators[3] = "\\/";
+const char xrDecompressor::s_Separators[4] = "\\/";
 
 xrDecompressor::xrDecompressor(const char* outDir)
     : m_OutDir(outDir)
@@ -123,20 +123,30 @@ void xrDecompressor::Decompress()
 {
     FS.load_all_unloaded_archives();
 
+    CTimer timer = {};
+    printf("Start Decompress\n");
+    timer.Start();
+
     auto files = FS.file_list_open("$game_data$");
     VERIFY(files);
 
     printf("%d file(s) to decompress in %d archive(s)\n\n", (int)files->size(), (int)FS.m_archives.size());
 
-    for (auto File : *files)
-    {
-        if (int err = ExtractFile(File))
+    xr_parallel_for(xr_blocked_range<size_t>(0, files->size()), [&](const xr_blocked_range<size_t>& r)
         {
-            printf("[ERROR] Failed to extract %s: %d", File, err);
-            FS.file_list_close(files);
-            return;
+            for (size_t i = r.begin(); i < r.end(); ++i)
+            {
+                const char* File = (*files)[i];
+                if (int err = ExtractFile(File))
+                {
+                    printf("[ERROR] Failed to extract %s: %d", File, err);
+                }
+            }
         }
-    }
+    );
+
+    printf("End Decompress: %d\n", timer.GetElapsed_ms());
+    Msg("End Decompress: %d\n", timer.GetElapsed_ms());
 
     FS.file_list_close(files);
 

diff --git a/src/utils/xrCompress/xrDecompress.h b/src/utils/xrCompress/xrDecompress.h
@@ -2,10 +2,10 @@
 
 class xrDecompressor
 {
-    static const char s_Separators[3];
+    static const char s_Separators[4];
     const char *m_OutDir;
 
-    xr_hash_map<xr_string, xr_string> m_PathCache;
+    tbb::concurrent_unordered_map<xr_string, xr_string> m_PathCache;
     virtual int ExtractFile(const char *filename);
     virtual const char * CreatePath(const char *path);
     virtual int CreateDir(const char *base, const char *path);

diff --git a/src/xrCore/_thread_types.h b/src/xrCore/_thread_types.h
@@ -4,6 +4,7 @@
 #include <tbb/parallel_for.h>
 #include <tbb/blocked_range.h>
 #include <tbb/parallel_for_each.h>
+#include <tbb/concurrent_unordered_map.h>
 
 #include <atomic>
 
@@ -13,6 +14,9 @@ using xr_task_group = tbb::task_group;
 template <typename T>
 using xr_blocked_range = tbb::blocked_range<T>;
 
+template <typename T, typename U>
+using xr_concurrent_unordered_map = tbb::concurrent_unordered_map<T, U>;
+
 template<typename BlockRange, typename Body>
 inline void xr_parallel_for(BlockRange Range, Body Functor)
 {