benchmark pure arrow vs fastformat

.gitignore

@@ -70,3 +70,9 @@ docs/_build/
 
 # Pyenv
 .python-version
+
+# PyO3 config
+.pyo3_config
+
+# UV
+uv.lock

examples/dora-benchmark/.gitignore

@@ -0,0 +1,73 @@
+/target
+/out
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+.pytest_cache/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+.venv/
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+include/
+man/
+venv/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+pip-selfcheck.json
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+.DS_Store
+
+# Sphinx documentation
+docs/_build/
+
+# PyCharm
+.idea/
+
+# VSCode
+.vscode/
+
+# Pyenv
+.python-version

examples/dora-benchmark/dataflow-fastformat.yml

@@ -0,0 +1,14 @@
+nodes:
+  - id: sender-fastformat
+    build: cargo build --manifest-path ./sender-fastformat/Cargo.toml --release
+    path: ./sender-fastformat/target/release/dora-benchmark-sender-fastformat
+    outputs:
+      - latency
+      - throughput
+
+  - id: receiver-fastformat
+    build: cargo build --manifest-path ./receiver-fastformat/Cargo.toml --release
+    path: ./receiver-fastformat/target/release/dora-benchmark-receiver-fastformat
+    inputs:
+      latency: sender-fastformat/latency
+      throughput: sender-fastformat/throughput

examples/dora-benchmark/dataflow-raw.yml

@@ -0,0 +1,14 @@
+nodes:
+  - id: sender-raw
+    build: cargo build --manifest-path ./sender-raw/Cargo.toml --release
+    path: ./sender-raw/target/release/dora-benchmark-sender-raw
+    outputs:
+      - latency
+      - throughput
+
+  - id: receiver-raw
+    build: cargo build --manifest-path ./receiver-raw/Cargo.toml --release
+    path: ./receiver-raw/target/release/dora-benchmark-receiver-raw
+    inputs:
+      latency: sender-raw/latency
+      throughput: sender-raw/throughput

examples/dora-benchmark/dataflow.yml

@@ -0,0 +1,28 @@
+nodes:
+  - id: sender-raw
+    build: cargo build --manifest-path ./sender-raw/Cargo.toml --release
+    path: ./sender-raw/target/release/dora-benchmark-sender-raw
+    outputs:
+      - latency
+      - throughput
+
+  - id: receiver-raw
+    build: cargo build --manifest-path ./receiver-raw/Cargo.toml --release
+    path: ./receiver-raw/target/release/dora-benchmark-receiver-raw
+    inputs:
+      latency: sender-raw/latency
+      throughput: sender-raw/throughput
+
+  - id: sender-fastformat
+    build: cargo build --manifest-path ./sender-fastformat/Cargo.toml --release
+    path: ./sender-fastformat/target/release/dora-benchmark-sender-fastformat
+    outputs:
+      - latency
+      - throughput
+
+  - id: receiver-fastformat
+    build: cargo build --manifest-path ./receiver-fastformat/Cargo.toml --release
+    path: ./receiver-fastformat/target/release/dora-benchmark-receiver-fastformat
+    inputs:
+      latency: sender-fastformat/latency
+      throughput: sender-fastformat/throughput

examples/dora-benchmark/main.rs

@@ -0,0 +1,88 @@
+use std::path::Path;
+
+use eyre::{Context, Result};
+
+fn main() -> Result<()> {
+    let root = Path::new(env!("CARGO_MANIFEST_DIR"));
+    std::env::set_current_dir(root.join(file!()).parent().unwrap())
+        .wrap_err("failed to set working dir")?;
+
+    let dataflow = Path::new("dataflow.yml");
+
+    destroy_dora()?;
+    spawn_dora()?;
+    build_dataflow(dataflow)?;
+    start_dataflow(dataflow)?;
+
+    Ok(())
+}
+
+fn destroy_dora() -> eyre::Result<()> {
+    let cargo = std::env::var("CARGO_HOME").unwrap();
+    let dora = Path::new(&cargo).join("bin").join("dora");
+
+    let mut cmd = std::process::Command::new(&dora);
+
+    cmd.arg("destroy");
+
+    if cmd.status().wrap_err("failed to destroy dora")?.success() {
+        println!("dora destroyed successfully");
+    } else {
+        println!("dora destroy failed");
+    }
+
+    Ok(())
+}
+
+fn spawn_dora() -> eyre::Result<()> {
+    let cargo = std::env::var("CARGO_HOME").unwrap();
+    let dora = Path::new(&cargo).join("bin").join("dora");
+
+    let mut cmd = std::process::Command::new(&dora);
+
+    cmd.arg("up");
+
+    if cmd.status().wrap_err("failed to spawn dora")?.success() {
+        println!("dora spawned successfully");
+    } else {
+        println!("dora spawn failed");
+    }
+
+    Ok(())
+}
+
+fn build_dataflow(dataflow: &Path) -> eyre::Result<()> {
+    let cargo = std::env::var("CARGO_HOME").unwrap();
+    let dora = Path::new(&cargo).join("bin").join("dora");
+
+    let mut cmd = std::process::Command::new(&dora);
+
+    cmd.arg("build");
+    cmd.arg("--").arg(dataflow);
+
+    if cmd.status().wrap_err("failed to build dataflow")?.success() {
+        println!("dataflow built successfully");
+    } else {
+        println!("dataflow build failed");
+    }
+
+    Ok(())
+}
+
+fn start_dataflow(dataflow: &Path) -> eyre::Result<()> {
+    let cargo = std::env::var("CARGO_HOME").unwrap();
+    let dora = Path::new(&cargo).join("bin").join("dora");
+
+    let mut cmd = std::process::Command::new(&dora);
+
+    cmd.arg("start");
+    cmd.arg("--").arg(dataflow);
+
+    if cmd.status().wrap_err("failed to start dataflow")?.success() {
+        println!("dataflow executed successfully");
+    } else {
+        println!("dataflow failed");
+    }
+
+    Ok(())
+}

examples/dora-benchmark/out/benchmark-all1/log_receiver-fastformat.txt

@@ -0,0 +1,12 @@
+Latency:
+size 0xfd200 : 631.22µs
+size 0x2a3000: 734.287µs
+size 0x5eec00: 1.119144ms
+size 0x17bb000: 2.084592ms
+Throughput:
+size 0xfd200 : 179 messages per second
+size 0x2a3000: 1858 messages per second
+size 0x5eec00: 608 messages per second
+Input `latency` was closed
+Input `throughput` was closed
+size 0x17bb000: 155 messages per second

examples/dora-benchmark/out/benchmark-all1/log_receiver-raw.txt

@@ -0,0 +1,12 @@
+Latency:
+size 0xfd200 : 510.034µs
+size 0x2a3000: 632.592µs
+size 0x5eec00: 796.755µs
+size 0x17bb000: 1.776238ms
+Throughput:
+size 0xfd200 : 1817 messages per second
+size 0x2a3000: 698 messages per second
+size 0x5eec00: 586 messages per second
+Input `latency` was closed
+Input `throughput` was closed
+size 0x17bb000: 160 messages per second

examples/dora-benchmark/out/benchmark-all2/log_receiver-fastformat.txt

@@ -0,0 +1,12 @@
+Latency:
+size 0xfd200 : 668.844µs
+size 0x2a3000: 784.455µs
+size 0x5eec00: 1.186606ms
+size 0x17bb000: 2.030822ms
+Throughput:
+size 0xfd200 : 176 messages per second
+size 0x2a3000: 1876 messages per second
+size 0x5eec00: 603 messages per second
+Input `latency` was closed
+Input `throughput` was closed
+size 0x17bb000: 155 messages per second

examples/dora-benchmark/out/benchmark-all2/log_receiver-raw.txt

@@ -0,0 +1,12 @@
+Latency:
+size 0xfd200 : 554.816µs
+size 0x2a3000: 663.627µs
+size 0x5eec00: 856.833µs
+size 0x17bb000: 1.715494ms
+Throughput:
+size 0xfd200 : 1972 messages per second
+size 0x2a3000: 662 messages per second
+size 0x5eec00: 535 messages per second
+Input `latency` was closed
+Input `throughput` was closed
+size 0x17bb000: 157 messages per second

examples/dora-benchmark/out/benchmark-ff1/log_receiver-fastformat.txt

@@ -0,0 +1,12 @@
+Latency:
+size 0xfd200 : 871.699µs
+size 0x2a3000: 710.012µs
+size 0x5eec00: 725.182µs
+size 0x17bb000: 678.061µs
+Throughput:
+size 0xfd200 : 2349 messages per second
+size 0x2a3000: 548 messages per second
+size 0x5eec00: 508 messages per second
+Input `latency` was closed
+Input `throughput` was closed
+size 0x17bb000: 152 messages per second

examples/dora-benchmark/out/benchmark-ff2/log_receiver-fastformat.txt

@@ -0,0 +1,12 @@
+Latency:
+size 0xfd200 : 842.827µs
+size 0x2a3000: 745.078µs
+size 0x5eec00: 754.759µs
+size 0x17bb000: 731.324µs
+Throughput:
+size 0xfd200 : 1973 messages per second
+size 0x2a3000: 549 messages per second
+size 0x5eec00: 529 messages per second
+Input `latency` was closed
+Input `throughput` was closed
+size 0x17bb000: 149 messages per second

examples/dora-benchmark/out/benchmark-raw1/log_receiver-raw.txt

@@ -0,0 +1,12 @@
+Latency:
+size 0xfd200 : 604.005µs
+size 0x2a3000: 638.604µs
+size 0x5eec00: 670.105µs
+size 0x17bb000: 663.552µs
+Throughput:
+size 0xfd200 : 2500 messages per second
+size 0x2a3000: 730 messages per second
+size 0x5eec00: 505 messages per second
+Input `latency` was closed
+Input `throughput` was closed
+size 0x17bb000: 155 messages per second

examples/dora-benchmark/out/benchmark-raw2/log_receiver-raw.txt

@@ -0,0 +1,12 @@
+Latency:
+size 0xfd200 : 757.939µs
+size 0x2a3000: 620.078µs
+size 0x5eec00: 656.879µs
+size 0x17bb000: 660.14µs
+Throughput:
+size 0xfd200 : 1937 messages per second
+size 0x2a3000: 700 messages per second
+size 0x5eec00: 522 messages per second
+Input `latency` was closed
+Input `throughput` was closed
+size 0x17bb000: 153 messages per second