diff --git a/crates/build/re_types_builder/src/codegen/rust/serializer.rs b/crates/build/re_types_builder/src/codegen/rust/serializer.rs index d5208b9bf543..11d3facc4b8c 100644 --- a/crates/build/re_types_builder/src/codegen/rust/serializer.rs +++ b/crates/build/re_types_builder/src/codegen/rust/serializer.rs @@ -585,57 +585,59 @@ fn quote_arrow_field_serializer( DataType::Utf8 => { // NOTE: We need values for all slots, regardless of what the validity says, // hence `unwrap_or_default`. - let (quoted_transparent_mapping, quoted_transparent_length) = - if inner_is_arrow_transparent { - let inner_obj = inner_obj.as_ref().unwrap(); - let is_tuple_struct = is_tuple_struct_from_obj(inner_obj); - let quoted_data_dst = format_ident!( - "{}", - if is_tuple_struct { - "data0" - } else { - inner_obj.fields[0].name.as_str() - } - ); - let quoted_member_accessor = if is_tuple_struct { - quote!(0) + let (quoted_member_accessor, quoted_transparent_length) = if inner_is_arrow_transparent + { + let inner_obj = inner_obj.as_ref().unwrap(); + let is_tuple_struct = is_tuple_struct_from_obj(inner_obj); + let quoted_data_dst = format_ident!( + "{}", + if is_tuple_struct { + "data0" } else { - quote!(#quoted_data_dst) - }; - - ( - quote! { - .flat_map(|datum| { - datum.#quoted_member_accessor.into_arrow2_buffer() - }) - }, - quote! { - .map(|datum| { - datum.#quoted_member_accessor.len() - }) - }, - ) + inner_obj.fields[0].name.as_str() + } + ); + let quoted_member_accessor = if is_tuple_struct { + quote!(.0) } else { - ( - quote! { - .flat_map(|s| s.into_arrow2_buffer()) - }, - quote! { - .map(|datum| datum.len()) - }, - ) + quote!(.#quoted_data_dst) }; + ( + quoted_member_accessor.clone(), + quote! { + .map(|datum| { + datum #quoted_member_accessor.len() + }) + }, + ) + } else { + ( + quote! {}, + quote! { + .map(|datum| datum.len()) + }, + ) + }; + let inner_data_and_offsets = if elements_are_nullable { quote! { let offsets = arrow::buffer::OffsetBuffer::::from_lengths( #data_src.iter().map(|opt| opt.as_ref() #quoted_transparent_length .unwrap_or_default()) ); + // Offsets is always non-empty. The last element is the total length of buffer we need. + // We want this capacity in order to allocate exactly as much memory as we need. + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + + let mut buffer_builder = arrow::array::builder::BufferBuilder::::new(capacity); // NOTE: Flattening to remove the guaranteed layer of nullability: we don't care // about it while building the backing buffer since it's all offsets driven. - let inner_data: arrow::buffer::Buffer = - #data_src.into_iter().flatten() #quoted_transparent_mapping.collect(); + for data in #data_src.iter().flatten() { + buffer_builder.append_slice(data #quoted_member_accessor.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); } } else { quote! { @@ -643,8 +645,16 @@ fn quote_arrow_field_serializer( #data_src.iter() #quoted_transparent_length ); - let inner_data: arrow::buffer::Buffer = - #data_src.into_iter() #quoted_transparent_mapping.collect(); + // Offsets is always non-empty. The last element is the total length of buffer we need. + // We want this capacity in order to allocate exactly as much memory as we need. + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + + let mut buffer_builder = arrow::array::builder::BufferBuilder::::new(capacity); + for data in &#data_src { + buffer_builder.append_slice(data #quoted_member_accessor.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); } }; diff --git a/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs b/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs index 2dde6fbb57c1..15257961b4dd 100644 --- a/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs +++ b/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs @@ -104,11 +104,15 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), ); - let inner_data: arrow::buffer::Buffer = entity_path - .into_iter() - .flatten() - .flat_map(|datum| datum.0.into_arrow2_buffer()) - .collect(); + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in entity_path.iter().flatten() { + buffer_builder.append_slice(data.0.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { StringArray::new_unchecked( @@ -137,11 +141,14 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), ); - let inner_data: arrow::buffer::Buffer = component - .into_iter() - .flatten() - .flat_map(|datum| datum.0.into_arrow2_buffer()) - .collect(); + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in component.iter().flatten() { + buffer_builder.append_slice(data.0.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { diff --git a/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs b/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs index 43d8bec361a1..db84346e598b 100644 --- a/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs +++ b/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs @@ -135,10 +135,15 @@ impl ::re_types_core::Loggable for SelectedColumns { let offsets = arrow::buffer::OffsetBuffer::::from_lengths( time_columns_inner_data.iter().map(|datum| datum.0.len()), ); - let inner_data: arrow::buffer::Buffer = time_columns_inner_data - .into_iter() - .flat_map(|datum| datum.0.into_arrow2_buffer()) - .collect(); + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in &time_columns_inner_data { + buffer_builder.append_slice(data.0.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { StringArray::new_unchecked( diff --git a/crates/store/re_types/src/blueprint/datatypes/utf8list.rs b/crates/store/re_types/src/blueprint/datatypes/utf8list.rs index 0e2bf517c123..c8a00d1be0d3 100644 --- a/crates/store/re_types/src/blueprint/datatypes/utf8list.rs +++ b/crates/store/re_types/src/blueprint/datatypes/utf8list.rs @@ -75,10 +75,15 @@ impl ::re_types_core::Loggable for Utf8List { let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0_inner_data.iter().map(|datum| datum.len()), ); - let inner_data: arrow::buffer::Buffer = data0_inner_data - .into_iter() - .flat_map(|s| s.into_arrow2_buffer()) - .collect(); + + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in &data0_inner_data { + buffer_builder.append_slice(data.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { diff --git a/crates/store/re_types/src/datatypes/annotation_info.rs b/crates/store/re_types/src/datatypes/annotation_info.rs index fc5903aecce8..72959487214b 100644 --- a/crates/store/re_types/src/datatypes/annotation_info.rs +++ b/crates/store/re_types/src/datatypes/annotation_info.rs @@ -118,11 +118,15 @@ impl ::re_types_core::Loggable for AnnotationInfo { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), ); - let inner_data: arrow::buffer::Buffer = label - .into_iter() - .flatten() - .flat_map(|datum| datum.0.into_arrow2_buffer()) - .collect(); + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in label.iter().flatten() { + buffer_builder.append_slice(data.0.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { StringArray::new_unchecked(offsets, inner_data, label_validity) diff --git a/crates/store/re_types/src/datatypes/tensor_data.rs b/crates/store/re_types/src/datatypes/tensor_data.rs index 315248747bee..0c28d9ff6285 100644 --- a/crates/store/re_types/src/datatypes/tensor_data.rs +++ b/crates/store/re_types/src/datatypes/tensor_data.rs @@ -193,10 +193,14 @@ impl ::re_types_core::Loggable for TensorData { let offsets = arrow::buffer::OffsetBuffer::::from_lengths( names_inner_data.iter().map(|datum| datum.len()), ); - let inner_data: arrow::buffer::Buffer = names_inner_data - .into_iter() - .flat_map(|s| s.into_arrow2_buffer()) - .collect(); + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in &names_inner_data { + buffer_builder.append_slice(data.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { StringArray::new_unchecked( diff --git a/crates/store/re_types/src/datatypes/utf8pair.rs b/crates/store/re_types/src/datatypes/utf8pair.rs index 928fb06e5fe6..38d450665af0 100644 --- a/crates/store/re_types/src/datatypes/utf8pair.rs +++ b/crates/store/re_types/src/datatypes/utf8pair.rs @@ -88,11 +88,15 @@ impl ::re_types_core::Loggable for Utf8Pair { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), ); - let inner_data: arrow::buffer::Buffer = first - .into_iter() - .flatten() - .flat_map(|datum| datum.0.into_arrow2_buffer()) - .collect(); + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in first.iter().flatten() { + buffer_builder.append_slice(data.0.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { StringArray::new_unchecked(offsets, inner_data, first_validity) @@ -117,11 +121,16 @@ impl ::re_types_core::Loggable for Utf8Pair { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), ); - let inner_data: arrow::buffer::Buffer = second - .into_iter() - .flatten() - .flat_map(|datum| datum.0.into_arrow2_buffer()) - .collect(); + + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in second.iter().flatten() { + buffer_builder.append_slice(data.0.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { StringArray::new_unchecked(offsets, inner_data, second_validity) diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer10.rs b/crates/store/re_types/src/testing/components/affix_fuzzer10.rs index 204f429d70fd..bbca1295e226 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer10.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer10.rs @@ -67,11 +67,14 @@ impl ::re_types_core::Loggable for AffixFuzzer10 { .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), ); - let inner_data: arrow::buffer::Buffer = data0 - .into_iter() - .flatten() - .flat_map(|s| s.into_arrow2_buffer()) - .collect(); + + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = arrow::array::builder::BufferBuilder::::new(capacity); + for data in data0.iter().flatten() { + buffer_builder.append_slice(data.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer12.rs b/crates/store/re_types/src/testing/components/affix_fuzzer12.rs index fbc078b5beb8..b4676ed1aaf7 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer12.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer12.rs @@ -80,10 +80,15 @@ impl ::re_types_core::Loggable for AffixFuzzer12 { let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0_inner_data.iter().map(|datum| datum.len()), ); - let inner_data: arrow::buffer::Buffer = data0_inner_data - .into_iter() - .flat_map(|s| s.into_arrow2_buffer()) - .collect(); + + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in &data0_inner_data { + buffer_builder.append_slice(data.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer13.rs b/crates/store/re_types/src/testing/components/affix_fuzzer13.rs index 9342d264d038..8937fe3bb284 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer13.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer13.rs @@ -80,10 +80,15 @@ impl ::re_types_core::Loggable for AffixFuzzer13 { let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0_inner_data.iter().map(|datum| datum.len()), ); - let inner_data: arrow::buffer::Buffer = data0_inner_data - .into_iter() - .flat_map(|s| s.into_arrow2_buffer()) - .collect(); + + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in &data0_inner_data { + buffer_builder.append_slice(data.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer9.rs b/crates/store/re_types/src/testing/components/affix_fuzzer9.rs index 59e32c0d2904..68c5ec93327a 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer9.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer9.rs @@ -67,11 +67,14 @@ impl ::re_types_core::Loggable for AffixFuzzer9 { .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), ); - let inner_data: arrow::buffer::Buffer = data0 - .into_iter() - .flatten() - .flat_map(|s| s.into_arrow2_buffer()) - .collect(); + + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = arrow::array::builder::BufferBuilder::::new(capacity); + for data in data0.iter().flatten() { + buffer_builder.append_slice(data.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs index ea42e58924af..13e6f4f041ae 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs @@ -188,11 +188,16 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { opt.as_ref().map(|datum| datum.len()).unwrap_or_default() }), ); - let inner_data: arrow::buffer::Buffer = single_string_required - .into_iter() - .flatten() - .flat_map(|s| s.into_arrow2_buffer()) - .collect(); + + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in single_string_required.iter().flatten() { + buffer_builder.append_slice(data.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { StringArray::new_unchecked( @@ -224,11 +229,16 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { opt.as_ref().map(|datum| datum.len()).unwrap_or_default() }), ); - let inner_data: arrow::buffer::Buffer = single_string_optional - .into_iter() - .flatten() - .flat_map(|s| s.into_arrow2_buffer()) - .collect(); + + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in single_string_optional.iter().flatten() { + buffer_builder.append_slice(data.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { StringArray::new_unchecked( @@ -319,11 +329,15 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { .iter() .map(|datum| datum.len()), ); - let inner_data: arrow::buffer::Buffer = - many_strings_required_inner_data - .into_iter() - .flat_map(|s| s.into_arrow2_buffer()) - .collect(); + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in &many_strings_required_inner_data { + buffer_builder.append_slice(data.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { StringArray::new_unchecked( @@ -375,11 +389,15 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { .iter() .map(|datum| datum.len()), ); - let inner_data: arrow::buffer::Buffer = - many_strings_optional_inner_data - .into_iter() - .flat_map(|s| s.into_arrow2_buffer()) - .collect(); + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in &many_strings_optional_inner_data { + buffer_builder.append_slice(data.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { StringArray::new_unchecked( diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs index e7b0af03f603..dd0c35d1ab74 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs @@ -121,11 +121,15 @@ impl ::re_types_core::Loggable for AffixFuzzer20 { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), ); - let inner_data: arrow::buffer::Buffer = s - .into_iter() - .flatten() - .flat_map(|datum| datum.0.into_arrow2_buffer()) - .collect(); + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in s.iter().flatten() { + buffer_builder.append_slice(data.0.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { StringArray::new_unchecked(offsets, inner_data, s_validity) diff --git a/crates/store/re_types/src/testing/datatypes/string_component.rs b/crates/store/re_types/src/testing/datatypes/string_component.rs index fc7b9f1cdb06..33519adf8643 100644 --- a/crates/store/re_types/src/testing/datatypes/string_component.rs +++ b/crates/store/re_types/src/testing/datatypes/string_component.rs @@ -61,11 +61,14 @@ impl ::re_types_core::Loggable for StringComponent { .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), ); - let inner_data: arrow::buffer::Buffer = data0 - .into_iter() - .flatten() - .flat_map(|s| s.into_arrow2_buffer()) - .collect(); + + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = arrow::array::builder::BufferBuilder::::new(capacity); + for data in data0.iter().flatten() { + buffer_builder.append_slice(data.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { diff --git a/crates/store/re_types_core/src/arrow_string.rs b/crates/store/re_types_core/src/arrow_string.rs index 492981a1ea3f..d4db7b085247 100644 --- a/crates/store/re_types_core/src/arrow_string.rs +++ b/crates/store/re_types_core/src/arrow_string.rs @@ -62,14 +62,15 @@ impl ArrowString { std::str::from_utf8(self.0.as_ref()).unwrap_or("INVALID UTF-8") } + /// The raw UTF8 bytes. #[inline] - pub fn into_arrow_buffer(self) -> arrow::buffer::Buffer { - self.0 + pub fn as_bytes(&self) -> &[u8] { + self.0.as_slice() } #[inline] - pub fn into_arrow2_buffer(self) -> arrow2::buffer::Buffer { - self.0.into() + pub fn into_arrow_buffer(self) -> arrow::buffer::Buffer { + self.0 } } diff --git a/crates/store/re_types_core/src/datatypes/entity_path.rs b/crates/store/re_types_core/src/datatypes/entity_path.rs index 6d0dcb557526..82500a6eb71f 100644 --- a/crates/store/re_types_core/src/datatypes/entity_path.rs +++ b/crates/store/re_types_core/src/datatypes/entity_path.rs @@ -62,11 +62,14 @@ impl crate::Loggable for EntityPath { .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), ); - let inner_data: arrow::buffer::Buffer = data0 - .into_iter() - .flatten() - .flat_map(|s| s.into_arrow2_buffer()) - .collect(); + + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = arrow::array::builder::BufferBuilder::::new(capacity); + for data in data0.iter().flatten() { + buffer_builder.append_slice(data.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { diff --git a/crates/store/re_types_core/src/datatypes/utf8.rs b/crates/store/re_types_core/src/datatypes/utf8.rs index 8b5a927b46fb..a55595ac30e0 100644 --- a/crates/store/re_types_core/src/datatypes/utf8.rs +++ b/crates/store/re_types_core/src/datatypes/utf8.rs @@ -62,11 +62,14 @@ impl crate::Loggable for Utf8 { .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), ); - let inner_data: arrow::buffer::Buffer = data0 - .into_iter() - .flatten() - .flat_map(|s| s.into_arrow2_buffer()) - .collect(); + + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = arrow::array::builder::BufferBuilder::::new(capacity); + for data in data0.iter().flatten() { + buffer_builder.append_slice(data.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { diff --git a/crates/store/re_types_core/src/datatypes/visible_time_range.rs b/crates/store/re_types_core/src/datatypes/visible_time_range.rs index c9855edf005e..d666d25e3a63 100644 --- a/crates/store/re_types_core/src/datatypes/visible_time_range.rs +++ b/crates/store/re_types_core/src/datatypes/visible_time_range.rs @@ -104,11 +104,15 @@ impl crate::Loggable for VisibleTimeRange { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), ); - let inner_data: arrow::buffer::Buffer = timeline - .into_iter() - .flatten() - .flat_map(|datum| datum.0.into_arrow2_buffer()) - .collect(); + #[allow(clippy::unwrap_used)] + let capacity = offsets.last().copied().unwrap() as usize; + let mut buffer_builder = + arrow::array::builder::BufferBuilder::::new(capacity); + for data in timeline.iter().flatten() { + buffer_builder.append_slice(data.0.as_bytes()); + } + let inner_data: arrow::buffer::Buffer = buffer_builder.finish(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] as_array_ref(unsafe { StringArray::new_unchecked(offsets, inner_data, timeline_validity)