Skip to content

Commit

Permalink
Fix performance regression due to extra arrow round-tripping (#8684)
Browse files Browse the repository at this point in the history
  • Loading branch information
emilk authored Jan 14, 2025
1 parent ecdc4d7 commit 3906792
Show file tree
Hide file tree
Showing 4 changed files with 19 additions and 5 deletions.
18 changes: 16 additions & 2 deletions crates/store/re_chunk/src/chunk.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,20 @@ impl ChunkComponents {
.map(|la| la.into())
}

#[inline]
pub fn insert_descriptor_arrow2(
&mut self,
component_desc: ComponentDescriptor,
list_array: Arrow2ListArray<i32>,
) -> Option<Arrow2ListArray<i32>> {
// TODO(cmc): revert me
let component_desc = component_desc.untagged();
self.0
.entry(component_desc.component_name)
.or_default()
.insert(component_desc, list_array)
}

/// Returns all list arrays for the given component name.
///
/// I.e semantically equivalent to `get("MyComponent:*.*")`
Expand Down Expand Up @@ -181,7 +195,7 @@ impl FromIterator<(ComponentDescriptor, Arrow2ListArray<i32>)> for ChunkComponen
let mut this = Self::default();
{
for (component_desc, list_array) in iter {
this.insert_descriptor(component_desc, list_array.into());
this.insert_descriptor_arrow2(component_desc, list_array);
}
}
this
Expand Down Expand Up @@ -956,7 +970,7 @@ impl Chunk {
list_array: Arrow2ListArray<i32>,
) -> ChunkResult<()> {
self.components
.insert_descriptor(component_desc, list_array.into());
.insert_descriptor_arrow2(component_desc, list_array);
self.sanity_check()
}

Expand Down
2 changes: 1 addition & 1 deletion crates/store/re_chunk/src/merge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ impl Chunk {
let components = {
let mut per_name = ChunkComponents::default();
for (component_desc, list_array) in components {
per_name.insert_descriptor(component_desc.clone(), list_array.into());
per_name.insert_descriptor_arrow2(component_desc.clone(), list_array);
}
per_name
};
Expand Down
2 changes: 1 addition & 1 deletion crates/store/re_chunk/src/migration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ impl Chunk {
}

for (desc, list_array) in components_patched {
chunk.components.insert_descriptor(desc, list_array.into());
chunk.components.insert_descriptor_arrow2(desc, list_array);
}

chunk
Expand Down
2 changes: 1 addition & 1 deletion crates/store/re_chunk/src/transport.rs
Original file line number Diff line number Diff line change
Expand Up @@ -705,7 +705,7 @@ impl Chunk {
let component_desc = TransportChunk::component_descriptor_from_field(field);

if components
.insert_descriptor(component_desc, column.clone().into())
.insert_descriptor_arrow2(component_desc, column.clone())
.is_some()
{
return Err(ChunkError::Malformed {
Expand Down

0 comments on commit 3906792

Please sign in to comment.