Port component storage to arrow-rs (#8725)

* This is a huge piece of #3741 This makes our store run 100% on arrow-rs. arrow2 is now relegated to the margins: * IPC * `re_types_builder` * Some legacy methods on `Loggable`, `Archetype` etc
rerun-io · Jan 21, 2025 · 67f93c1 · 67f93c1
1 parent 4948ee8
commit 67f93c1
Show file tree

Hide file tree

Showing 72 changed files with 606 additions and 1,926 deletions.
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
@@ -49,7 +49,7 @@ NOTE: `.rrd` files do not yet guarantee any backwards or forwards compatibility.
 
 [Apache Arrow](https://arrow.apache.org/) is a language-independent columnar memory format for arbitrary data. We use it to encode the log data when transmitting it over the network or storing it in an `.rrd` file. We also use it in our in-RAM data store, [`re_chunk_store`](crates/store/re_chunk_store/README.md).
 
-In Rust, we use the [`arrow2` crate](https://crates.io/crates/arrow2).
+In Rust, we use the [`arrow` crate](https://crates.io/crates/arrow).
 
 ### `wgpu`
 

diff --git a/Cargo.lock b/Cargo.lock
@@ -356,9 +356,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
 
 [[package]]
 name = "arrow"
-version = "53.2.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4caf25cdc4a985f91df42ed9e9308e1adbcd341a31a72605c697033fcef163e3"
+checksum = "eaf3437355979f1e93ba84ba108c38be5767713051f3c8ffbf07c094e2e61f9f"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -376,9 +376,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-arith"
-version = "53.2.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91f2dfd1a7ec0aca967dfaa616096aec49779adc8eccec005e2f5e4111b1192a"
+checksum = "31dce77d2985522288edae7206bffd5fc4996491841dda01a13a58415867e681"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -391,25 +391,25 @@ dependencies = [
 
 [[package]]
 name = "arrow-array"
-version = "53.2.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d39387ca628be747394890a6e47f138ceac1aa912eab64f02519fed24b637af8"
+checksum = "2d45fe6d3faed0435b7313e59a02583b14c6c6339fa7729e94c32a20af319a79"
 dependencies = [
  "ahash",
  "arrow-buffer",
  "arrow-data",
  "arrow-schema",
  "chrono",
  "half",
- "hashbrown 0.14.5",
+ "hashbrown 0.15.2",
  "num",
 ]
 
 [[package]]
 name = "arrow-buffer"
-version = "53.2.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e51e05228852ffe3eb391ce7178a0f97d2cf80cc6ef91d3c4a6b3cb688049ec"
+checksum = "2b02656a35cc103f28084bc80a0159668e0a680d919cef127bd7e0aaccb06ec1"
 dependencies = [
  "bytes",
  "half",
@@ -418,9 +418,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-cast"
-version = "53.2.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d09aea56ec9fa267f3f3f6cdab67d8a9974cbba90b3aa38c8fe9d0bb071bd8c1"
+checksum = "c73c6233c5b5d635a56f6010e6eb1ab9e30e94707db21cea03da317f67d84cf3"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -438,9 +438,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "53.2.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b98ae0af50890b494cebd7d6b04b35e896205c1d1df7b29a6272c5d0d0249ef5"
+checksum = "b7f2861ffa86f107b8ab577d86cff7c7a490243eabe961ba1e1af4f27542bb79"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
@@ -460,9 +460,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "53.2.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ed91bdeaff5a1c00d28d8f73466bcb64d32bbd7093b5a30156b4b9f4dba3eee"
+checksum = "0270dc511f11bb5fa98a25020ad51a99ca5b08d8a8dfbd17503bb9dba0388f0b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -474,9 +474,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ord"
-version = "53.2.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2883d7035e0b600fb4c30ce1e50e66e53d8656aa729f2bfa4b51d359cf3ded52"
+checksum = "c6f202a879d287099139ff0d121e7f55ae5e0efe634b8cf2106ebc27a8715dee"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -489,9 +489,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-row"
-version = "53.2.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "552907e8e587a6fde4f8843fd7a27a576a260f65dab6c065741ea79f633fc5be"
+checksum = "a8f936954991c360ba762dff23f5dda16300774fafd722353d9683abd97630ae"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -503,18 +503,18 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "53.2.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "539ada65246b949bd99ffa0881a9a15a4a529448af1a07a9838dd78617dafab1"
+checksum = "9579b9d8bce47aa41389fe344f2c6758279983b7c0ebb4013e283e3e91bb450e"
 dependencies = [
  "bitflags 2.6.0",
 ]
 
 [[package]]
 name = "arrow-select"
-version = "53.2.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6259e566b752da6dceab91766ed8b2e67bf6270eb9ad8a6e07a33c1bede2b125"
+checksum = "7471ba126d0b0aaa24b50a36bc6c25e4e74869a1fd1a5553357027a0b1c8d1f1"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -526,9 +526,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-string"
-version = "53.2.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3179ccbd18ebf04277a095ba7321b93fd1f774f18816bd5f6b3ce2f594edb6c"
+checksum = "72993b01cb62507b06f1fb49648d7286c8989ecfabdb7b77a750fcb54410731b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5607,7 +5607,6 @@ dependencies = [
  "arrow-schema",
  "bytemuck",
  "chrono",
- "comfy-table",
  "dyn-clone",
  "either",
  "ethnum",
@@ -5627,7 +5626,6 @@ version = "0.22.0-alpha.1+dev"
 dependencies = [
  "arrow",
  "itertools 0.13.0",
- "re_arrow2",
  "re_log",
  "re_tracing",
 ]
@@ -5691,7 +5689,6 @@ version = "0.22.0-alpha.1+dev"
 dependencies = [
  "arrow",
  "half",
- "re_arrow2",
  "smallvec",
 ]
 
@@ -5722,11 +5719,11 @@ dependencies = [
  "criterion",
  "crossbeam",
  "document-features",
+ "half",
  "itertools 0.13.0",
  "mimalloc",
  "nohash-hasher",
  "rand",
- "re_arrow2",
  "re_arrow_util",
  "re_byte_size",
  "re_error",
@@ -5760,7 +5757,6 @@ dependencies = [
  "once_cell",
  "parking_lot",
  "rand",
- "re_arrow2",
  "re_arrow_util",
  "re_byte_size",
  "re_chunk",
@@ -5782,6 +5778,7 @@ dependencies = [
 name = "re_chunk_store_ui"
 version = "0.22.0-alpha.1+dev"
 dependencies = [
+ "arrow",
  "egui",
  "egui_extras",
  "itertools 0.13.0",
@@ -5860,11 +5857,11 @@ dependencies = [
  "arrow",
  "crossbeam",
  "image",
+ "itertools 0.13.0",
  "notify",
  "once_cell",
  "parking_lot",
  "rayon",
- "re_arrow2",
  "re_build_info",
  "re_build_tools",
  "re_chunk",
@@ -5940,7 +5937,6 @@ dependencies = [
  "itertools 0.13.0",
  "nohash-hasher",
  "rayon",
- "re_arrow2",
  "re_arrow_util",
  "re_chunk",
  "re_chunk_store",
@@ -6271,7 +6267,6 @@ dependencies = [
  "parking_lot",
  "paste",
  "rand",
- "re_arrow2",
  "re_arrow_util",
  "re_byte_size",
  "re_chunk",
@@ -6339,7 +6334,6 @@ dependencies = [
  "pathdiff",
  "pollster 0.4.0",
  "profiling",
- "re_arrow2",
  "re_build_tools",
  "re_error",
  "re_log",
@@ -6568,7 +6562,6 @@ dependencies = [
  "once_cell",
  "ply-rs",
  "rayon",
- "re_arrow2",
  "re_build_tools",
  "re_byte_size",
  "re_format",
@@ -6631,7 +6624,6 @@ dependencies = [
  "itertools 0.13.0",
  "nohash-hasher",
  "once_cell",
- "re_arrow2",
  "re_arrow_util",
  "re_byte_size",
  "re_case",

diff --git a/Cargo.toml b/Cargo.toml
@@ -153,7 +153,7 @@ ahash = "0.8"
 anyhow = { version = "1.0", default-features = false }
 argh = "0.1.12"
 array-init = "2.1"
-arrow = { version = "53.1", default-features = false }
+arrow = { version = "53.4", default-features = false }
 arrow2 = { package = "re_arrow2", version = "0.18", features = ["arrow"] }
 async-executor = "1.0"
 backtrace = "0.3"

diff --git a/crates/store/re_chunk/Cargo.toml b/crates/store/re_chunk/Cargo.toml
@@ -49,14 +49,9 @@ re_types_core.workspace = true
 ahash.workspace = true
 anyhow.workspace = true
 arrow.workspace = true
-arrow2 = { workspace = true, features = [
-  "arrow",
-  "compute_concatenate",
-  "compute_filter",
-  "compute_take",
-] }
 bytemuck.workspace = true
 document-features.workspace = true
+half.workspace = true
 itertools.workspace = true
 nohash-hasher.workspace = true
 rand = { workspace = true, features = ["std_rng"] }

diff --git a/crates/store/re_chunk/src/batcher.rs b/crates/store/re_chunk/src/batcher.rs
@@ -798,7 +798,7 @@ impl PendingRow {
             re_tracing::profile_scope!("iterate per timeline set");
 
             // Then we split the micro batches even further -- one sub-batch per unique set of datatypes.
-            let mut per_datatype_set: IntMap<u64 /* Arrow2Datatype set */, Vec<Self>> =
+            let mut per_datatype_set: IntMap<u64 /* ArrowDatatype set */, Vec<Self>> =
                 Default::default();
             {
                 re_tracing::profile_scope!("compute datatype sets");