Codegen/IDL 2: introduce re_types_builder

.github/workflows/labels.yml

@@ -29,4 +29,4 @@ jobs:
         with:
           mode: minimum
           count: 1
-          labels: "📊 analytics, 🪳 bug, 🧑‍💻 dev experience, dependencies, 📖 documentation, 💬 discussion, examples, 📉 performance, 🐍 python API, ⛃ re_datastore, 📺 re_viewer, 🔺 re_renderer, 🚜 refactor, ⛴ release, 🦀 rust SDK, 🔨 testing, ui, 🕸️ web"
+          labels: "📊 analytics, 🪳 bug, codegen/idl, 🧑‍💻 dev experience, dependencies, 📖 documentation, 💬 discussion, examples, 📉 performance, 🐍 python API, ⛃ re_datastore, 📺 re_viewer, 🔺 re_renderer, 🚜 refactor, ⛴ release, 🦀 rust SDK, 🔨 testing, ui, 🕸️ web"

.gitignore

@@ -6,7 +6,7 @@
 **/target_wasm
 
 # Python virtual environment:
-/venv
+**/venv
 
 # Python build artifacts:
 __pycache__

.vscode/extensions.json

@@ -1,19 +1,20 @@
 {
-    // See https://go.microsoft.com/fwlink/?LinkId=827846
-    // for the documentation about the extensions.json format
-    "recommendations": [
-        "charliermarsh.ruff",
-        "github.vscode-github-actions",
-        "ms-python.python",
-        "ms-vsliveshare.vsliveshare",
-        "polymeilex.wgsl",
-        "rust-lang.rust-analyzer",
-        "serayuzgur.crates",
-        "streetsidesoftware.code-spell-checker",
-        "tamasfe.even-better-toml",
-        "vadimcn.vscode-lldb",
-        "wayou.vscode-todo-highlight",
-        "webfreak.debug",
-        "zxh404.vscode-proto3",
-    ]
+  // See https://go.microsoft.com/fwlink/?LinkId=827846
+  // for the documentation about the extensions.json format
+  "recommendations": [
+    "charliermarsh.ruff",
+    "github.vscode-github-actions",
+    "ms-python.python",
+    "ms-vsliveshare.vsliveshare",
+    "polymeilex.wgsl",
+    "rust-lang.rust-analyzer",
+    "serayuzgur.crates",
+    "streetsidesoftware.code-spell-checker",
+    "tamasfe.even-better-toml",
+    "vadimcn.vscode-lldb",
+    "wayou.vscode-todo-highlight",
+    "webfreak.debug",
+    "zxh404.vscode-proto3",
+    "gaborv.flatbuffers"
+  ]
 }

.vscode/settings.json

@@ -27,6 +27,7 @@
         "bindgroup",
         "colormap",
         "emath",
+        "flatbuffers",
         "framebuffer",
         "hoverable",
         "ilog",

Cargo.toml

@@ -56,6 +56,7 @@ re_tensor_ops = { path = "crates/re_tensor_ops", version = "0.7.0-alpha.0", defa
 re_time_panel = { path = "crates/re_time_panel", version = "=0.7.0-alpha.0", default-features = false }
 re_tracing = { path = "crates/re_tracing", version = "0.7.0-alpha.0", default-features = false }
 re_tuid = { path = "crates/re_tuid", version = "0.7.0-alpha.0", default-features = false }
+re_types_builder = { path = "crates/re_types_builder", version = "=0.7.0-alpha.0", default-features = false }
 re_ui = { path = "crates/re_ui", version = "0.7.0-alpha.0", default-features = false }
 re_viewer = { path = "crates/re_viewer", version = "0.7.0-alpha.0", default-features = false }
 re_viewer_context = { path = "crates/re_viewer_context", version = "0.7.0-alpha.0", default-features = false }
@@ -109,6 +110,7 @@ time = { version = "0.3", default-features = false, features = [
 ] }
 tinyvec = { version = "1.6", features = ["alloc", "rustc_1_55"] }
 tokio = { version = "1.24", default-features = false }
+unindent = "0.1"
 vec1 = "1.8"
 web-time = "0.2.0"
 wgpu = { version = "0.16.1" }

_typos.toml

@@ -13,6 +13,7 @@ extend-exclude = [
 [default.extend-words]
 lod = "lod" # level-of-detail
 teh = "teh" # part of @teh-cmc
+ND = "ND"   # np.NDArray
 
 # American English:
 grey = "gray"

crates/re_build_tools/Cargo.toml

@@ -20,4 +20,7 @@ all-features = true
 anyhow.workspace = true
 cargo_metadata = "0.15"
 glob = "0.3"
+sha2 = "0.10"
 time = { workspace = true, features = ["formatting"] }
+unindent = "0.1"
+walkdir = "2.0"

crates/re_build_tools/src/hashing.rs

@@ -0,0 +1,181 @@
+use std::fmt::Write;
+use std::path::{Path, PathBuf};
+use std::{fs, io};
+
+use anyhow::Context as _;
+use sha2::{Digest, Sha256};
+
+use crate::{rerun_if_changed, rerun_if_changed_or_doesnt_exist};
+
+// ---
+
+fn encode_hex(bytes: &[u8]) -> String {
+    let mut s = String::with_capacity(bytes.len() * 2);
+    for &b in bytes {
+        write!(&mut s, "{b:02x}").unwrap();
+    }
+    s
+}
+
+/// Walks the directory at `path` in filename order.
+///
+/// If `extensions` is specified, only files with the right extensions will be iterated.
+/// Specified extensions should _not_ include the leading dot, e.g. `fbs` rather than `.fbs`.
+pub fn iter_dir<'a>(
+    path: impl AsRef<Path>,
+    extensions: Option<&'a [&'a str]>,
+) -> impl Iterator<Item = PathBuf> + 'a {
+    fn filter(entry: &walkdir::DirEntry, extensions: Option<&[&str]>) -> bool {
+        let is_dir = entry.file_type().is_dir();
+        let is_interesting = extensions.map_or(true, |extensions| {
+            extensions.iter().any(|ext| {
+                entry
+                    .path()
+                    .extension()
+                    .map_or(false, |ext2| *ext == ext2.to_string_lossy())
+            })
+        });
+        is_dir || is_interesting
+    }
+
+    let path = path.as_ref();
+    walkdir::WalkDir::new(path)
+        .sort_by_file_name()
+        .into_iter()
+        .filter_entry(move |entry| filter(entry, extensions))
+        .filter_map(|entry| entry.ok())
+        .filter_map(|entry| entry.file_type().is_file().then(|| entry.into_path()))
+}
+
+/// Given a file path, computes the sha256 hash of its contents and returns an hexadecimal string
+/// for it.
+///
+/// This will automatically emit a `rerun-if-changed` clause for the specified file.
+///
+/// Panics if the file doesn't exist.
+pub fn compute_file_hash(path: impl AsRef<Path>) -> String {
+    let mut hasher = Sha256::new();
+
+    let path = path.as_ref();
+    let mut file = fs::File::open(path)
+        .with_context(|| format!("couldn't open {path:?}"))
+        .unwrap();
+    io::copy(&mut file, &mut hasher)
+        .with_context(|| format!("couldn't copy from {path:?}"))
+        .unwrap();
+
+    rerun_if_changed(path);
+
+    encode_hex(hasher.finalize().as_slice())
+}
+
+/// Given a directory path, computes the sha256 hash of the accumulated contents of all of its
+/// files (ordered by filename), and returns an hexadecimal string for it.
+///
+/// This includes files in sub-directories (i.e. it's recursive).
+///
+/// This will automatically emit a `rerun-if-changed` clause for all the files that were hashed.
+///
+/// If `extensions` is specified, only files with the right extensions will be iterated.
+/// Specified extensions should _not_ include the leading dot, e.g. `fbs` rather than `.fbs`.
+pub fn compute_dir_hash<'a>(path: impl AsRef<Path>, extensions: Option<&'a [&'a str]>) -> String {
+    let mut hasher = Sha256::new();
+
+    let path = path.as_ref();
+    for filepath in iter_dir(path, extensions) {
+        let mut file = fs::File::open(&filepath)
+            .with_context(|| format!("couldn't open {filepath:?}"))
+            .unwrap();
+        io::copy(&mut file, &mut hasher)
+            .with_context(|| format!("couldn't copy from {filepath:?}"))
+            .unwrap();
+
+        rerun_if_changed(path);
+    }
+
+    encode_hex(hasher.finalize().as_slice())
+}
+
+/// Given a crate name, computes the sha256 hash of its source code (ordered by filename) and
+/// returns an hexadecimal string for it.
+///
+/// This includes the source code of all its direct and indirect dependencies.
+///
+/// This will automatically emit a `rerun-if-changed` clause for all the files that were hashed.
+pub fn compute_crate_hash(pkg_name: impl AsRef<str>) -> String {
+    use cargo_metadata::{CargoOpt, MetadataCommand};
+    let metadata = MetadataCommand::new()
+        .features(CargoOpt::AllFeatures)
+        .exec()
+        .unwrap();
+
+    let pkg_name = pkg_name.as_ref();
+    let mut files = Default::default();
+
+    let pkgs = crate::Packages::from_metadata(&metadata);
+    pkgs.track_implicit_dep(pkg_name, &mut files);
+
+    let mut files = files.into_iter().collect::<Vec<_>>();
+    files.sort();
+
+    let hashes = files.into_iter().map(compute_file_hash).collect::<Vec<_>>();
+    let hashes = hashes.iter().map(|s| s.as_str()).collect::<Vec<_>>();
+
+    compute_strings_hash(&hashes)
+}
+
+/// Given a bunch of strings, computes the sha256 hash of their contents (in the order they
+/// were passed in) and returns an hexadecimal string for it.
+pub fn compute_strings_hash(strs: &[&str]) -> String {
+    let mut hasher = Sha256::new();
+
+    for s in strs {
+        hasher.update(s);
+    }
+
+    encode_hex(hasher.finalize().as_slice())
+}
+
+/// Writes the given `hash` at the specified `path`.
+///
+/// `hash` should have been computed using of the methods in this module: [`compute_file_hash`],
+/// [`compute_dir_hash`], [`compute_crate_hash`].
+///
+/// Panics on I/O errors.
+///
+/// Use [`read_versioning_hash`] to read it back.
+pub fn write_versioning_hash(path: impl AsRef<Path>, hash: impl AsRef<str>) {
+    let path = path.as_ref();
+    let hash = hash.as_ref();
+
+    let contents = unindent::unindent(&format!(
+        "
+        # This is a sha256 hash for all direct and indirect dependencies of this crate's build script.
+        # It can be safely removed at anytime to force the build script to run again.
+        # Check out build.rs to see how it's computed.
+        {hash}
+        "
+    ));
+    std::fs::write(path, contents.trim())
+        .with_context(|| format!("couldn't write to {path:?}"))
+        .unwrap();
+}
+
+/// Reads back a versioning hash that was written with [`write_versioning_hash`].
+///
+/// This will automatically emit a `rerun-if-changed` clause for the specified filepath.
+///
+/// Returns `None` on error.
+pub fn read_versioning_hash(path: impl AsRef<Path>) -> Option<String> {
+    let path = path.as_ref();
+
+    // NOTE: It's important we trigger if the file doesn't exist, as this means the user explicitly
+    // deleted the versioning file, i.e. they're trying to force a rebuild.
+    rerun_if_changed_or_doesnt_exist(path);
+
+    std::fs::read_to_string(path).ok().and_then(|contents| {
+        contents
+            .lines()
+            .find_map(|line| (!line.trim().starts_with('#')).then(|| line.trim().to_owned()))
+    })
+}

crates/re_build_tools/src/lib.rs

@@ -8,9 +8,16 @@ use anyhow::Context as _;
 
 use std::process::Command;
 
+mod hashing;
 mod rebuild_detector;
 
-pub use rebuild_detector::{
+pub(crate) use self::rebuild_detector::Packages;
+
+pub use self::hashing::{
+    compute_crate_hash, compute_dir_hash, compute_file_hash, compute_strings_hash, iter_dir,
+    read_versioning_hash, write_versioning_hash,
+};
+pub use self::rebuild_detector::{
     get_and_track_env_var, is_tracked_env_var_set, rebuild_if_crate_changed, rerun_if_changed,
     rerun_if_changed_glob, rerun_if_changed_or_doesnt_exist, write_file_if_necessary,
 };

crates/re_build_tools/src/rebuild_detector.rs

@@ -105,7 +105,7 @@ pub fn write_file_if_necessary(
 
 // ---
 
-struct Packages<'a> {
+pub struct Packages<'a> {
     pkgs: HashMap<&'a str, &'a Package>,
 }
 

crates/re_renderer/Cargo.toml

@@ -92,7 +92,7 @@ winit = "0.28.1"
 zip = { version = "0.6", default-features = false, features = ["deflate"] }
 
 # For tests:
-unindent = "0.1"
+unindent.workspace = true
 
 # native
 [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]