diff --git a/crates/store/re_chunk/src/builder.rs b/crates/store/re_chunk/src/builder.rs index 279aa625a223..e08e20132fab 100644 --- a/crates/store/re_chunk/src/builder.rs +++ b/crates/store/re_chunk/src/builder.rs @@ -3,7 +3,7 @@ use itertools::Itertools; use nohash_hasher::IntMap; use re_log_types::{EntityPath, TimeInt, TimePoint, Timeline}; -use re_types_core::{AsComponents, ComponentBatch, ComponentDescriptor}; +use re_types_core::{AsComponents, ComponentBatch, ComponentDescriptor, SerializedComponentBatch}; use crate::{arrow_util, chunk::ChunkComponents, Chunk, ChunkId, ChunkResult, RowId, TimeColumn}; @@ -121,14 +121,8 @@ impl ChunkBuilder { timepoint: impl Into, as_components: &dyn AsComponents, ) -> Self { - let batches = as_components.as_component_batches(); - self.with_component_batches( - row_id, - timepoint, - batches - .iter() - .map(|batch| batch as &dyn re_types_core::ComponentBatch), - ) + let batches = as_components.as_serialized_batches(); + self.with_serialized_batches(row_id, timepoint, batches) } /// Add a row's worth of data by serializing a single [`ComponentBatch`]. @@ -193,6 +187,59 @@ impl ChunkBuilder { ) } + /// Add a row's worth of data by serializing a single [`ComponentBatch`]. + #[inline] + pub fn with_serialized_batch( + self, + row_id: RowId, + timepoint: impl Into, + component_batch: SerializedComponentBatch, + ) -> Self { + self.with_row( + row_id, + timepoint, + [(component_batch.descriptor, component_batch.array)], + ) + } + + /// Add a row's worth of data by serializing many [`ComponentBatch`]es. + #[inline] + pub fn with_serialized_batches( + self, + row_id: RowId, + timepoint: impl Into, + component_batches: impl IntoIterator, + ) -> Self { + self.with_row( + row_id, + timepoint, + component_batches + .into_iter() + .map(|component_batch| (component_batch.descriptor, component_batch.array)), + ) + } + + /// Add a row's worth of data by serializing many sparse [`ComponentBatch`]es. + #[inline] + pub fn with_sparse_serialized_batches( + self, + row_id: RowId, + timepoint: impl Into, + component_batches: impl IntoIterator< + Item = (ComponentDescriptor, Option), + >, + ) -> Self { + self.with_sparse_row( + row_id, + timepoint, + component_batches + .into_iter() + .map(|(component_desc, component_batch)| { + (component_desc, component_batch.map(|batch| batch.array)) + }), + ) + } + /// Builds and returns the final [`Chunk`]. /// /// The arrow datatype of each individual column will be guessed by inspecting the data. diff --git a/crates/store/re_entity_db/tests/clear.rs b/crates/store/re_entity_db/tests/clear.rs index c72944b49b82..dbbb4a78927b 100644 --- a/crates/store/re_entity_db/tests/clear.rs +++ b/crates/store/re_entity_db/tests/clear.rs @@ -122,14 +122,7 @@ fn clears() -> anyhow::Result<()> { let timepoint = TimePoint::from_iter([(timeline_frame, 10)]); let clear = Clear::flat(); let chunk = Chunk::builder(entity_path_parent.clone()) - .with_component_batches( - row_id, - timepoint, - clear - .as_component_batches() - .iter() - .map(|b| b as &dyn re_types_core::ComponentBatch), - ) + .with_serialized_batches(row_id, timepoint, clear.as_serialized_batches()) .build()?; db.add_chunk(&Arc::new(chunk))?; @@ -163,14 +156,7 @@ fn clears() -> anyhow::Result<()> { let timepoint = TimePoint::from_iter([(timeline_frame, 10)]); let clear = Clear::recursive(); let chunk = Chunk::builder(entity_path_parent.clone()) - .with_component_batches( - row_id, - timepoint, - clear - .as_component_batches() - .iter() - .map(|b| b as &dyn re_types_core::ComponentBatch), - ) + .with_serialized_batches(row_id, timepoint, clear.as_serialized_batches()) .build()?; db.add_chunk(&Arc::new(chunk))?; @@ -351,13 +337,10 @@ fn clears_respect_index_order() -> anyhow::Result<()> { let clear = Clear::recursive(); let chunk = Chunk::builder(entity_path.clone()) - .with_component_batches( + .with_serialized_batches( row_id1, // older row id! timepoint.clone(), - clear - .as_component_batches() - .iter() - .map(|b| b as &dyn re_types_core::ComponentBatch), + clear.as_serialized_batches(), ) .build()?; @@ -378,13 +361,10 @@ fn clears_respect_index_order() -> anyhow::Result<()> { let clear = Clear::recursive(); let chunk = Chunk::builder(entity_path.clone()) - .with_component_batches( + .with_serialized_batches( row_id3, // newer row id! timepoint.clone(), - clear - .as_component_batches() - .iter() - .map(|b| b as &dyn re_types_core::ComponentBatch), + clear.as_serialized_batches(), ) .build()?; diff --git a/crates/store/re_types_core/src/as_components.rs b/crates/store/re_types_core/src/as_components.rs index 9e88fa0c950f..498d7c139221 100644 --- a/crates/store/re_types_core/src/as_components.rs +++ b/crates/store/re_types_core/src/as_components.rs @@ -1,6 +1,5 @@ use crate::{ - ComponentBatch, ComponentBatchCowWithDescriptor, LoggableBatch as _, ResultExt as _, - SerializationResult, + ComponentBatch, ComponentBatchCowWithDescriptor, SerializationResult, SerializedComponentBatch, }; /// Describes the interface for interpreting an object as a bundle of [`Component`]s. @@ -20,6 +19,8 @@ use crate::{ /// [Custom Data Loader]: https://github.com/rerun-io/rerun/blob/latest/examples/rust/custom_data_loader /// [`Component`]: [crate::Component] pub trait AsComponents { + /// Deprecated. Do not use. See [`AsComponents::as_serialized_batches`] instead. + /// /// Exposes the object's contents as a set of [`ComponentBatch`]s. /// /// This is the main mechanism for easily extending builtin archetypes or even writing @@ -34,7 +35,37 @@ pub trait AsComponents { // // NOTE: Don't bother returning a CoW here: we need to dynamically discard optional components // depending on their presence (or lack thereof) at runtime anyway. - fn as_component_batches(&self) -> Vec>; + #[deprecated(since = "0.22.0", note = "use as_serialized_batches instead")] + #[allow(clippy::unimplemented)] // temporary, this method is about to be replaced + fn as_component_batches(&self) -> Vec> { + // Eagerly serialized archetypes simply cannot implement this. + // + // This method only exist while we are in the process of making all existing archetypes + // eagerly serialized, at which point it'll be removed. + unimplemented!() + } + + /// Exposes the object's contents as a set of [`SerializedComponentBatch`]es. + /// + /// This is the main mechanism for easily extending builtin archetypes or even writing + /// fully custom ones. + /// Have a look at our [Custom Data Loader] example to learn more about extending archetypes. + /// + /// Implementers of [`AsComponents`] get one last chance to override the tags in the + /// [`ComponentDescriptor`], see [`SerializedComponentBatch::with_descriptor_override`]. + /// + /// [Custom Data Loader]: https://github.com/rerun-io/rerun/blob/latest/docs/snippets/all/tutorials/custom_data.rs + /// [`ComponentDescriptor`]: [crate::ComponentDescriptor] + // + // NOTE: Don't bother returning a CoW here: we need to dynamically discard optional components + // depending on their presence (or lack thereof) at runtime anyway. + fn as_serialized_batches(&self) -> Vec { + #[allow(deprecated)] // that's the whole point + self.as_component_batches() + .into_iter() + .filter_map(|batch| batch.serialized()) + .collect() + } // --- @@ -48,20 +79,15 @@ pub trait AsComponents { fn to_arrow( &self, ) -> SerializationResult> { - self.as_component_batches() + self.as_serialized_batches() .into_iter() .map(|comp_batch| { - comp_batch - .to_arrow() - .map(|array| { - let field = arrow::datatypes::Field::new( - comp_batch.name().to_string(), - array.data_type().clone(), - false, - ); - (field, array) - }) - .with_context(comp_batch.name()) + let field = arrow::datatypes::Field::new( + comp_batch.descriptor.component_name.to_string(), + comp_batch.array.data_type().clone(), + false, + ); + Ok((field, comp_batch.array)) }) .collect() } @@ -77,20 +103,15 @@ pub trait AsComponents { &self, ) -> SerializationResult)>> { - self.as_component_batches() + self.as_serialized_batches() .into_iter() .map(|comp_batch| { - comp_batch - .to_arrow2() - .map(|array| { - let field = arrow2::datatypes::Field::new( - comp_batch.name().to_string(), - array.data_type().clone(), - false, - ); - (field, array) - }) - .with_context(comp_batch.name()) + let field = arrow2::datatypes::Field::new( + comp_batch.descriptor.component_name.to_string(), + comp_batch.array.data_type().clone().into(), + false, + ); + Ok((field, comp_batch.array.into())) }) .collect() } @@ -103,106 +124,96 @@ fn assert_object_safe() { impl AsComponents for dyn ComponentBatch { #[inline] - fn as_component_batches(&self) -> Vec> { - vec![ComponentBatchCowWithDescriptor::new(self)] + fn as_serialized_batches(&self) -> Vec { + self.serialized().into_iter().collect() } } impl AsComponents for [&dyn ComponentBatch; N] { #[inline] - fn as_component_batches(&self) -> Vec> { - self.iter() - .map(|batch| ComponentBatchCowWithDescriptor::new(*batch)) - .collect() + fn as_serialized_batches(&self) -> Vec { + self.iter().filter_map(|batch| batch.serialized()).collect() } } impl AsComponents for [Box; N] { #[inline] - fn as_component_batches(&self) -> Vec> { - self.iter() - .map(|batch| ComponentBatchCowWithDescriptor::new(&**batch)) - .collect() + fn as_serialized_batches(&self) -> Vec { + self.iter().filter_map(|batch| batch.serialized()).collect() } } impl AsComponents for Vec<&dyn ComponentBatch> { #[inline] - fn as_component_batches(&self) -> Vec> { - self.iter() - .map(|batch| ComponentBatchCowWithDescriptor::new(*batch)) - .collect() + fn as_serialized_batches(&self) -> Vec { + self.iter().filter_map(|batch| batch.serialized()).collect() } } impl AsComponents for Vec> { #[inline] - fn as_component_batches(&self) -> Vec> { - self.iter() - .map(|batch| ComponentBatchCowWithDescriptor::new(&**batch)) - .collect() + fn as_serialized_batches(&self) -> Vec { + self.iter().filter_map(|batch| batch.serialized()).collect() } } -impl AsComponents for [AS] { +impl AsComponents for SerializedComponentBatch { #[inline] - fn as_component_batches(&self) -> Vec> { - self.iter() - .flat_map(|as_components| as_components.as_component_batches()) - .collect() + fn as_serialized_batches(&self) -> Vec { + vec![self.clone()] } } impl AsComponents for [AS; N] { #[inline] - fn as_component_batches(&self) -> Vec> { + fn as_serialized_batches(&self) -> Vec { self.iter() - .flat_map(|as_components| as_components.as_component_batches()) + .flat_map(|as_components| as_components.as_serialized_batches()) .collect() } } impl AsComponents for [&dyn AsComponents; N] { #[inline] - fn as_component_batches(&self) -> Vec> { + fn as_serialized_batches(&self) -> Vec { self.iter() - .flat_map(|as_components| as_components.as_component_batches()) + .flat_map(|as_components| as_components.as_serialized_batches()) .collect() } } impl AsComponents for [Box; N] { #[inline] - fn as_component_batches(&self) -> Vec> { + fn as_serialized_batches(&self) -> Vec { self.iter() - .flat_map(|as_components| as_components.as_component_batches()) + .flat_map(|as_components| as_components.as_serialized_batches()) .collect() } } impl AsComponents for Vec { #[inline] - fn as_component_batches(&self) -> Vec> { + fn as_serialized_batches(&self) -> Vec { self.iter() - .flat_map(|as_components| as_components.as_component_batches()) + .flat_map(|as_components| as_components.as_serialized_batches()) .collect() } } impl AsComponents for Vec<&dyn AsComponents> { #[inline] - fn as_component_batches(&self) -> Vec> { + fn as_serialized_batches(&self) -> Vec { self.iter() - .flat_map(|as_components| as_components.as_component_batches()) + .flat_map(|as_components| as_components.as_serialized_batches()) .collect() } } impl AsComponents for Vec> { #[inline] - fn as_component_batches(&self) -> Vec> { + fn as_serialized_batches(&self) -> Vec { self.iter() - .flat_map(|as_components| as_components.as_component_batches()) + .flat_map(|as_components| as_components.as_serialized_batches()) .collect() } } @@ -293,10 +304,9 @@ mod tests { use arrow::array::{ types::UInt32Type, Array as ArrowArray, PrimitiveArray as ArrowPrimitiveArray, }; + use itertools::Itertools; use similar_asserts::assert_eq; - use crate::LoggableBatch; - #[derive(Clone, Copy, Debug, PartialEq, Eq, bytemuck::Pod, bytemuck::Zeroable)] #[repr(transparent)] pub struct MyColor(pub u32); @@ -356,24 +366,21 @@ mod tests { } #[test] - fn single_ascomponents_howto() -> anyhow::Result<()> { + fn single_ascomponents_howto() { let (red, _, _, _) = data(); let got = { let red = &red as &dyn crate::ComponentBatch; - let got: Result, _> = (&[red] as &dyn crate::AsComponents) - .as_component_batches() + (&[red] as &dyn crate::AsComponents) + .as_serialized_batches() .into_iter() - .map(|batch| batch.to_arrow()) - .collect(); - got? + .map(|batch| batch.array) + .collect_vec() }; let expected = vec![ Arc::new(ArrowPrimitiveArray::::from(vec![red.0])) as Arc, ]; assert_eq!(&expected, &got); - - Ok(()) } #[test] @@ -390,24 +397,21 @@ mod tests { } #[test] - fn single_ascomponents_wrapped_howto() -> anyhow::Result<()> { + fn single_ascomponents_wrapped_howto() { let (red, _, _, _) = data(); let got = { let red = &red as &dyn crate::ComponentBatch; - let got: Result, _> = (&[red] as &dyn crate::AsComponents) - .as_component_batches() + (&[red] as &dyn crate::AsComponents) + .as_serialized_batches() .into_iter() - .map(|batch| batch.to_arrow()) - .collect(); - got? + .map(|batch| batch.array) + .collect_vec() }; let expected = vec![ Arc::new(ArrowPrimitiveArray::::from(vec![red.0])) as Arc, ]; assert_eq!(&expected, &got); - - Ok(()) } #[test] @@ -424,19 +428,18 @@ mod tests { } #[test] - fn single_ascomponents_wrapped_many_howto() -> anyhow::Result<()> { + fn single_ascomponents_wrapped_many_howto() { let (red, green, blue, _) = data(); let got = { let red = &red as &dyn crate::ComponentBatch; let green = &green as &dyn crate::ComponentBatch; let blue = &blue as &dyn crate::ComponentBatch; - let got: Result, _> = (&[red, green, blue] as &dyn crate::AsComponents) - .as_component_batches() + (&[red, green, blue] as &dyn crate::AsComponents) + .as_serialized_batches() .into_iter() - .map(|batch| batch.to_arrow()) - .collect(); - got? + .map(|batch| batch.array) + .collect_vec() }; let expected = vec![ Arc::new(ArrowPrimitiveArray::::from(vec![red.0])) as Arc, @@ -444,8 +447,6 @@ mod tests { Arc::new(ArrowPrimitiveArray::::from(vec![blue.0])) as Arc, ]; assert_eq!(&expected, &got); - - Ok(()) } #[test] @@ -477,39 +478,35 @@ mod tests { } #[test] - fn many_ascomponents_wrapped_howto() -> anyhow::Result<()> { + fn many_ascomponents_wrapped_howto() { let (red, green, blue, colors) = data(); let got = { let colors = &colors as &dyn crate::ComponentBatch; - let got: Result, _> = (&[colors] as &dyn crate::AsComponents) - .as_component_batches() + (&[colors] as &dyn crate::AsComponents) + .as_serialized_batches() .into_iter() - .map(|batch| batch.to_arrow()) - .collect(); - got? + .map(|batch| batch.array) + .collect_vec() }; let expected = vec![Arc::new(ArrowPrimitiveArray::::from(vec![ red.0, green.0, blue.0, ])) as Arc]; assert_eq!(&expected, &got); - - Ok(()) } #[test] - fn many_ascomponents_wrapped_many_howto() -> anyhow::Result<()> { + fn many_ascomponents_wrapped_many_howto() { let (red, green, blue, colors) = data(); // Nothing out of the ordinary here, a collection of batches is indeed a collection of batches. let got = { let colors = &colors as &dyn crate::ComponentBatch; - let got: Result, _> = (&[colors, colors, colors] as &dyn crate::AsComponents) - .as_component_batches() + (&[colors, colors, colors] as &dyn crate::AsComponents) + .as_serialized_batches() .into_iter() - .map(|batch| batch.to_arrow()) - .collect(); - got? + .map(|batch| batch.array) + .collect_vec() }; let expected = vec![ Arc::new(ArrowPrimitiveArray::::from(vec![ @@ -523,7 +520,5 @@ mod tests { ])) as Arc, ]; assert_eq!(&expected, &got); - - Ok(()) } } diff --git a/crates/store/re_types_core/src/lib.rs b/crates/store/re_types_core/src/lib.rs index c631ebc35b65..aa012b7bcab3 100644 --- a/crates/store/re_types_core/src/lib.rs +++ b/crates/store/re_types_core/src/lib.rs @@ -52,6 +52,7 @@ pub use self::{ }, loggable_batch::{ ComponentBatch, ComponentBatchCow, ComponentBatchCowWithDescriptor, LoggableBatch, + SerializedComponentBatch, }, result::{ DeserializationError, DeserializationResult, ResultExt, SerializationError, diff --git a/crates/store/re_types_core/src/loggable_batch.rs b/crates/store/re_types_core/src/loggable_batch.rs index 72f23628f052..6c1c3cd2110f 100644 --- a/crates/store/re_types_core/src/loggable_batch.rs +++ b/crates/store/re_types_core/src/loggable_batch.rs @@ -1,6 +1,9 @@ use std::borrow::Cow; -use crate::{Component, ComponentDescriptor, ComponentName, Loggable, SerializationResult}; +use crate::{ + ArchetypeFieldName, ArchetypeName, Component, ComponentDescriptor, ComponentName, Loggable, + SerializationResult, +}; use arrow2::array::ListArray as Arrow2ListArray; @@ -55,6 +58,9 @@ pub trait ComponentBatch: LoggableBatch { fn descriptor(&self) -> Cow<'_, ComponentDescriptor>; // Wraps the current [`ComponentBatch`] with the given descriptor. + // + // TODO(cmc): This should probably go away, but we'll see about that once I start tackling + // partial updates themselves. fn with_descriptor( &self, descriptor: ComponentDescriptor, @@ -66,6 +72,69 @@ pub trait ComponentBatch: LoggableBatch { .with_descriptor_override(descriptor) } + /// Serializes the contents of this [`ComponentBatch`]. + /// + /// Once serialized, the data is ready to be logged into Rerun via the [`AsComponents`] trait. + /// + /// # Fallibility + /// + /// There are very few ways in which serialization can fail, all of which are very rare to hit + /// in practice. + /// One such example is trying to serialize data with more than 2^31 elements into a `ListArray`. + /// + /// For that reason, this method favors a nice user experience over error handling: errors will + /// merely be logged, not returned (except in debug builds, where all errors panic). + /// + /// See also [`ComponentBatch::try_serialized`]. + /// + /// [`AsComponents`]: [crate::AsComponents] + #[inline] + fn serialized(&self) -> Option { + match self.try_serialized() { + Ok(array) => Some(array), + + #[cfg(debug_assertions)] + Err(err) => { + panic!( + "failed to serialize data for {}: {}", + self.descriptor(), + re_error::format_ref(&err) + ) + } + + #[cfg(not(debug_assertions))] + Err(err) => { + re_log::error!( + descriptor = %self.descriptor(), + "failed to serialize data: {}", + re_error::format_ref(&err) + ); + None + } + } + } + + /// Serializes the contents of this [`ComponentBatch`]. + /// + /// Once serialized, the data is ready to be logged into Rerun via the [`AsComponents`] trait. + /// + /// # Fallibility + /// + /// There are very few ways in which serialization can fail, all of which are very rare to hit + /// in practice. + /// + /// For that reason, it generally makes sense to favor a nice user experience over error handling + /// in most cases, see [`ComponentBatch::serialized`]. + /// + /// [`AsComponents`]: [crate::AsComponents] + #[inline] + fn try_serialized(&self) -> SerializationResult { + Ok(SerializedComponentBatch { + array: self.to_arrow()?, + descriptor: self.descriptor().into_owned(), + }) + } + /// The fully-qualified name of this component batch, e.g. `rerun.components.Position2D`. /// /// This is a trivial but useful helper for `self.descriptor().component_name`. @@ -85,6 +154,89 @@ fn assert_component_batch_object_safe() { let _: &dyn LoggableBatch; } +/// The serialized contents of a [`ComponentBatch`] with associated [`ComponentDescriptor`]. +/// +/// This is what gets logged into Rerun: +/// * See [`ComponentBatch`] to easily serialize component data. +/// * See [`AsComponents`] for logging serialized data. +/// +/// [`AsComponents`]: [crate::AsComponents] +#[derive(Debug, Clone)] +pub struct SerializedComponentBatch { + pub array: arrow::array::ArrayRef, + + // TODO(cmc): Maybe Cow<> this one if it grows bigger. Or intern descriptors altogether, most likely. + pub descriptor: ComponentDescriptor, +} + +impl re_byte_size::SizeBytes for SerializedComponentBatch { + #[inline] + fn heap_size_bytes(&self) -> u64 { + let Self { array, descriptor } = self; + array.heap_size_bytes() + descriptor.heap_size_bytes() + } +} + +impl PartialEq for SerializedComponentBatch { + #[inline] + fn eq(&self, other: &Self) -> bool { + let Self { array, descriptor } = self; + + // Descriptor first! + *descriptor == other.descriptor && **array == *other.array + } +} + +impl SerializedComponentBatch { + #[inline] + pub fn new(array: arrow::array::ArrayRef, descriptor: ComponentDescriptor) -> Self { + Self { array, descriptor } + } + + #[inline] + pub fn with_descriptor_override(self, descriptor: ComponentDescriptor) -> Self { + Self { descriptor, ..self } + } + + /// Unconditionally sets the descriptor's `archetype_name` to the given one. + #[inline] + pub fn with_archetype_name(mut self, archetype_name: ArchetypeName) -> Self { + self.descriptor = self.descriptor.with_archetype_name(archetype_name); + self + } + + /// Unconditionally sets the descriptor's `archetype_field_name` to the given one. + #[inline] + pub fn with_archetype_field_name(mut self, archetype_field_name: ArchetypeFieldName) -> Self { + self.descriptor = self + .descriptor + .with_archetype_field_name(archetype_field_name); + self + } + + /// Sets the descriptor's `archetype_name` to the given one iff it's not already set. + #[inline] + pub fn or_with_archetype_name(mut self, archetype_name: impl Fn() -> ArchetypeName) -> Self { + self.descriptor = self.descriptor.or_with_archetype_name(archetype_name); + self + } + + /// Sets the descriptor's `archetype_field_name` to the given one iff it's not already set. + #[inline] + pub fn or_with_archetype_field_name( + mut self, + archetype_field_name: impl FnOnce() -> ArchetypeFieldName, + ) -> Self { + self.descriptor = self + .descriptor + .or_with_archetype_field_name(archetype_field_name); + self + } +} + +// TODO(cmc): All these crazy types are about to disappear. ComponentBatch should only live at the +// edge, and therefore not require all these crazy kinds of derivatives (require eager serialization). + /// Some [`ComponentBatch`], optionally with an overridden [`ComponentDescriptor`]. /// /// Used by implementers of [`crate::AsComponents`] to both efficiently expose their component data diff --git a/crates/top/re_sdk/src/lib.rs b/crates/top/re_sdk/src/lib.rs index 642e844ae903..5756313ef465 100644 --- a/crates/top/re_sdk/src/lib.rs +++ b/crates/top/re_sdk/src/lib.rs @@ -95,6 +95,7 @@ pub use re_types_core::{ ComponentBatchCowWithDescriptor, ComponentDescriptor, ComponentName, DatatypeName, DeserializationError, DeserializationResult, GenericIndicatorComponent, Loggable, LoggableBatch, NamedIndicatorComponent, SerializationError, SerializationResult, + SerializedComponentBatch, }; pub use re_byte_size::SizeBytes; diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index 8771691ac47c..7333056146a4 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -1033,10 +1033,10 @@ impl RecordingStream { #[deprecated(since = "0.16.0", note = "use `log_static` instead")] #[doc(hidden)] #[inline] - pub fn log_timeless( + pub fn log_timeless( &self, ent_path: impl Into, - arch: &impl AsComponents, + arch: &AS, ) -> RecordingStreamResult<()> { self.log_static(ent_path, arch) } @@ -1063,10 +1063,10 @@ impl RecordingStream { /// [SDK Micro Batching]: https://www.rerun.io/docs/reference/sdk/micro-batching /// [component bundle]: [`AsComponents`] #[inline] - pub fn log_static( + pub fn log_static( &self, ent_path: impl Into, - as_components: &impl AsComponents, + as_components: &AS, ) -> RecordingStreamResult<()> { self.log_with_static(ent_path, true, as_components) } @@ -1074,11 +1074,11 @@ impl RecordingStream { #[deprecated(since = "0.16.0", note = "use `log_static` instead")] #[doc(hidden)] #[inline] - pub fn log_with_timeless( + pub fn log_with_timeless( &self, ent_path: impl Into, static_: bool, - arch: &impl AsComponents, + arch: &AS, ) -> RecordingStreamResult<()> { self.log_with_static(ent_path, static_, arch) } @@ -1113,14 +1113,11 @@ impl RecordingStream { as_components: &AS, ) -> RecordingStreamResult<()> { let row_id = RowId::new(); // Create row-id as early as possible. It has a timestamp and is used to estimate e2e latency. - self.log_component_batches_impl( + self.log_serialized_batches_impl( row_id, ent_path, static_, - as_components - .as_component_batches() - .iter() - .map(|any_comp_batch| any_comp_batch as &dyn re_types_core::ComponentBatch), + as_components.as_serialized_batches(), ) } @@ -1147,6 +1144,7 @@ impl RecordingStream { /// See [SDK Micro Batching] for more information. /// /// [SDK Micro Batching]: https://www.rerun.io/docs/reference/sdk/micro-batching + #[deprecated(since = "0.22.0", note = "use log_serialized_batches instead")] pub fn log_component_batches<'a>( &self, ent_path: impl Into, @@ -1196,6 +1194,79 @@ impl RecordingStream { Ok(()) } + /// Logs a set of [`SerializedComponentBatch`]es into Rerun. + /// + /// If `static_` is set to `true`, all timestamp data associated with this message will be + /// dropped right before sending it to Rerun. + /// Static data has no time associated with it, exists on all timelines, and unconditionally shadows + /// any temporal data of the same type. + /// + /// Otherwise, the data will be timestamped automatically based on the [`RecordingStream`]'s + /// internal clock. + /// See `RecordingStream::set_time_*` family of methods for more information. + /// + /// The number of instances will be determined by the longest batch in the bundle. + /// + /// The entity path can either be a string + /// (with special characters escaped, split on unescaped slashes) + /// or an [`EntityPath`] constructed with [`crate::entity_path`]. + /// See for more on entity paths. + /// + /// Internally, the stream will automatically micro-batch multiple log calls to optimize + /// transport. + /// See [SDK Micro Batching] for more information. + /// + /// [SDK Micro Batching]: https://www.rerun.io/docs/reference/sdk/micro-batching + /// + /// [`SerializedComponentBatch`]: [re_types_core::SerializedComponentBatch] + pub fn log_serialized_batches( + &self, + ent_path: impl Into, + static_: bool, + comp_batches: impl IntoIterator, + ) -> RecordingStreamResult<()> { + let row_id = RowId::new(); // Create row-id as early as possible. It has a timestamp and is used to estimate e2e latency. + self.log_serialized_batches_impl(row_id, ent_path, static_, comp_batches) + } + + // NOTE: For bw and fw compatibility reasons, we need our logging APIs to be fallible, even + // though they really aren't at the moment. + #[allow(clippy::unnecessary_wraps)] + fn log_serialized_batches_impl( + &self, + row_id: RowId, + entity_path: impl Into, + static_: bool, + comp_batches: impl IntoIterator, + ) -> RecordingStreamResult<()> { + if !self.is_enabled() { + return Ok(()); // silently drop the message + } + + let entity_path = entity_path.into(); + + let comp_batches: Vec<_> = comp_batches + .into_iter() + .map(|comp_batch| (comp_batch.descriptor, comp_batch.array)) + .collect(); + let components: IntMap<_, _> = comp_batches.into_iter().collect(); + + // NOTE: The timepoint is irrelevant, the `RecordingStream` will overwrite it using its + // internal clock. + let timepoint = TimePoint::default(); + + if !components.is_empty() { + let row = PendingRow { + row_id, + timepoint, + components, + }; + self.record_row(entity_path, row, !static_); + } + + Ok(()) + } + /// Logs the file at the given `path` using all [`re_data_loader::DataLoader`]s available. /// /// A single `path` might be handled by more than one loader. diff --git a/docs/snippets/all/descriptors/descr_builtin_component.rs b/docs/snippets/all/descriptors/descr_builtin_component.rs index 86e6bc52c6f6..dfb391127036 100644 --- a/docs/snippets/all/descriptors/descr_builtin_component.rs +++ b/docs/snippets/all/descriptors/descr_builtin_component.rs @@ -1,10 +1,11 @@ use rerun::{ChunkStore, ChunkStoreConfig, Component as _, ComponentDescriptor, VersionPolicy}; fn example(rec: &rerun::RecordingStream) -> Result<(), Box> { - rec.log_component_batches( + use rerun::ComponentBatch as _; + rec.log_serialized_batches( "data", true, - [&rerun::components::Position3D::new(1.0, 2.0, 3.0) as &dyn rerun::ComponentBatch], + [rerun::components::Position3D::new(1.0, 2.0, 3.0).try_serialized()?], )?; Ok(()) diff --git a/docs/snippets/all/descriptors/descr_custom_component.rs b/docs/snippets/all/descriptors/descr_custom_component.rs index 14e3f20505bb..a8bb1d0f9ee2 100644 --- a/docs/snippets/all/descriptors/descr_custom_component.rs +++ b/docs/snippets/all/descriptors/descr_custom_component.rs @@ -1,13 +1,14 @@ use rerun::{ChunkStore, ChunkStoreConfig, ComponentBatch, ComponentDescriptor, VersionPolicy}; fn example(rec: &rerun::RecordingStream) -> Result<(), Box> { - let positions = rerun::components::Position3D::new(1.0, 2.0, 3.0); - let positions = positions.with_descriptor(ComponentDescriptor { - archetype_name: Some("user.CustomArchetype".into()), - archetype_field_name: Some("custom_positions".into()), - component_name: "user.CustomPosition3D".into(), - }); - rec.log_component_batches("data", true, [&positions as &dyn rerun::ComponentBatch])?; + let positions = rerun::components::Position3D::new(1.0, 2.0, 3.0) + .try_serialized()? + .with_descriptor_override(ComponentDescriptor { + archetype_name: Some("user.CustomArchetype".into()), + archetype_field_name: Some("custom_positions".into()), + component_name: "user.CustomPosition3D".into(), + }); + rec.log_serialized_batches("data", true, [positions])?; Ok(()) } diff --git a/docs/snippets/all/tutorials/custom_data.rs b/docs/snippets/all/tutorials/custom_data.rs index 7b3d3fa00b17..2857159a2729 100644 --- a/docs/snippets/all/tutorials/custom_data.rs +++ b/docs/snippets/all/tutorials/custom_data.rs @@ -3,7 +3,7 @@ use rerun::{ demo_util::grid, external::{arrow, glam, re_types}, - ComponentBatch, + ComponentBatch, SerializedComponentBatch, }; // --- @@ -18,27 +18,22 @@ struct CustomPoints3D { } impl rerun::AsComponents for CustomPoints3D { - fn as_component_batches(&self) -> Vec> { + fn as_serialized_batches(&self) -> Vec { let indicator = rerun::NamedIndicatorComponent("user.CustomPoints3DIndicator".into()); self.points3d - .as_component_batches() + .as_serialized_batches() .into_iter() .chain( [ - Some(indicator.to_batch()), - self.confidences.as_ref().map(|batch| { - rerun::ComponentBatchCowWithDescriptor::new( - batch as &dyn rerun::ComponentBatch, - ) - // Optionally override the descriptor with extra information. - .with_descriptor_override( + indicator.serialized(), + self.confidences + .as_ref() + .and_then(|batch| batch.serialized()) + .map(|batch| + // Optionally override the descriptor with extra information. batch - .descriptor() - .into_owned() .or_with_archetype_name(|| "user.CustomPoints3D".into()) - .or_with_archetype_field_name(|| "confidences".into()), - ) - }), + .or_with_archetype_field_name(|| "confidences".into())), ] .into_iter() .flatten(), diff --git a/scripts/ci/check_large_files.py b/scripts/ci/check_large_files.py index b83948029e1c..52c35878f3c4 100755 --- a/scripts/ci/check_large_files.py +++ b/scripts/ci/check_large_files.py @@ -11,6 +11,7 @@ "crates/store/re_dataframe/src/query.rs", "crates/store/re_types/src/datatypes/tensor_buffer.rs", "crates/store/re_types/src/reflection/mod.rs", + "crates/top/re_sdk/src/recording_stream.rs", "crates/viewer/re_ui/data/Inter-Medium.otf", "docs/snippets/INDEX.md", "pixi.lock",