feat!: add arrow-rs features for buffer and array interop

.github/workflows/bench.yml

@@ -31,7 +31,7 @@ jobs:
             ${{ runner.os }}-cargo-${{ steps.rust-toolchain.outputs.cachekey }}-bench-
             ${{ runner.os }}-cargo-${{ steps.rust-toolchain.outputs.cachekey }}-
             ${{ runner.os }}-cargo-
-      - run: cargo bench --bench narrow -- --output-format=bencher | tee output.txt
+      - run: cargo bench --bench narrow --all-features -- --output-format=bencher | tee output.txt
       - uses: actions/upload-artifact@v3
         with:
           name: benchmark-results

.github/workflows/test.yml

@@ -7,13 +7,44 @@ permissions:
   contents: read
 
 jobs:
+  minimal:
+    name: Minimal
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        id: rust-toolchain
+      - uses: dtolnay/install@master
+        with:
+          crate: cargo-expand
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ steps.rust-toolchain.outputs.cachekey }}-minimal-${{ hashFiles('**/Cargo.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-${{ steps.rust-toolchain.outputs.cachekey }}-minimal-
+            ${{ runner.os }}-cargo-${{ steps.rust-toolchain.outputs.cachekey }}-
+            ${{ runner.os }}-cargo-
+      - run: cargo check --workspace --all-targets --no-default-features
+      - run: cargo test --workspace --all-targets --no-default-features
+      - run: cargo test --workspace --doc --no-default-features
+      - run: cargo clippy --workspace --all-targets --no-default-features -- -Dwarnings
+
   msrv:
     name: Minimum supported Rust version
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@1.67.0
+      - uses: dtolnay/rust-toolchain@1.70.0
         id: rust-toolchain
+      - uses: dtolnay/install@master
+        with:
+          crate: cargo-expand
       - uses: actions/cache@v3
         with:
           path: |
@@ -27,8 +58,9 @@ jobs:
             ${{ runner.os }}-cargo-${{ steps.rust-toolchain.outputs.cachekey }}-msrv-
             ${{ runner.os }}-cargo-${{ steps.rust-toolchain.outputs.cachekey }}-
             ${{ runner.os }}-cargo-
-      - run: cargo check --all --all-features
-
+      - run: cargo check --workspace --all-targets --all-features
+      - run: cargo test --all --all-targets --all-features
+      - run: cargo test --all --doc --all-features
   check:
     name: Check
     runs-on: ubuntu-latest
@@ -75,8 +107,8 @@ jobs:
       - uses: dtolnay/install@master
         with:
           crate: cargo-expand
-      - run: cargo test --all --all-targets --all-features
-      - run: cargo test --all --doc --all-features
+      - run: cargo test --workspace --all-targets --all-features
+      - run: cargo test --workspace --doc --all-features
 
   rustfmt:
     name: Rustfmt
@@ -110,7 +142,7 @@ jobs:
             ${{ runner.os }}-cargo-${{ steps.rust-toolchain.outputs.cachekey }}-clippy-
             ${{ runner.os }}-cargo-${{ steps.rust-toolchain.outputs.cachekey }}-
             ${{ runner.os }}-cargo-
-      - run: cargo clippy --all --all-targets --all-features -- -Dwarnings
+      - run: cargo clippy --workspace --all-targets --all-features -- -Dwarnings
 
   miri:
     name: Miri
@@ -133,6 +165,7 @@ jobs:
             ${{ runner.os }}-cargo-${{ steps.rust-toolchain.outputs.cachekey }}-
             ${{ runner.os }}-cargo-
       - run: cargo miri setup
+      - run: cargo miri test --no-default-features
       - run: cargo miri test --all-features
 
   coverage:
@@ -163,8 +196,8 @@ jobs:
       - uses: dtolnay/install@master
         with:
           crate: cargo-expand
-      - run: cargo build --all --all-targets --all-features
-      - run: cargo test --all --all-targets --all-features
+      - run: cargo build --workspace --all-targets --all-features
+      - run: cargo test --workspace --all-targets --all-features
         env:
           LLVM_PROFILE_FILE: "narrow-%p-%m.profraw"
       - name: Install grcov

Cargo.toml

@@ -1,7 +1,7 @@
 [workspace.package]
 authors = ["Matthijs Brobbel <[email protected]>"]
 edition = "2021"
-rust-version = "1.67.0"
+rust-version = "1.70.0"
 description = "An implementation of Apache Arrow"
 readme = "README.md"
 repository = "https://github.com/mbrobbel/narrow"
@@ -29,14 +29,21 @@ categories.workspace = true
 
 [features]
 default = []
+arrow-rs = ["dep:arrow-array", "dep:arrow-buffer", "dep:arrow-schema", "narrow-derive?/arrow-rs"]
 derive = ["dep:narrow-derive"]
 
 [dependencies]
+arrow-array = { git = "https://github.com/apache/arrow-rs", rev = "7fd2d42", optional = true }
+arrow-buffer = { git = "https://github.com/apache/arrow-rs", rev = "7fd2d42", optional = true }
+arrow-schema = { git = "https://github.com/apache/arrow-rs", rev = "7fd2d42", optional = true }
 narrow-derive = { path = "narrow-derive", version = "^0.3.4", optional = true }
 
 [dev-dependencies]
+arrow-cast = { git = "https://github.com/apache/arrow-rs", rev = "7fd2d42", default-features = false, features = ["prettyprint"] }
+bytes = "1.5.0"
 criterion = { version = "0.5.1", default-features = false }
 rand = { version = "0.8.5", default-features = false, features = ["small_rng"] }
+parquet = { git = "https://github.com/apache/arrow-rs", rev = "7fd2d42", features = ["arrow"] }
 
 [profile.bench]
 lto = true
@@ -45,3 +52,7 @@ codegen-units = 1
 [[bench]]
 name = "narrow"
 harness = false
+
+[[example]]
+name = "parquet"
+required-features = ["arrow-rs", "derive"]

README.md

@@ -16,7 +16,7 @@ This crate provides types to support reading and writing instances of abstract d
 
 ## Minimum supported Rust version
 
-The minimum supported Rust version for this crate is Rust 1.67.0.
+The minimum supported Rust version for this crate is Rust 1.70.0.
 
 ## License
 

examples/parquet.rs

@@ -0,0 +1,55 @@
+fn main() {
+    use arrow_array::RecordBatch;
+    use arrow_cast::pretty;
+    use bytes::Bytes;
+    use narrow::{array::StructArray, arrow::buffer_builder::ArrowBufferBuilder, ArrayType};
+    use parquet::arrow::{arrow_reader::ParquetRecordBatchReader, ArrowWriter};
+
+    #[derive(ArrayType, Default)]
+    struct Bar(Option<bool>);
+
+    #[derive(ArrayType, Default)]
+    struct Foo {
+        a: u32,
+        b: Option<u8>,
+        c: bool,
+        d: String,
+        e: Option<Vec<Option<bool>>>,
+        f: Bar,
+    }
+    let input = [
+        Foo {
+            a: 1,
+            b: Some(2),
+            c: true,
+            d: "hello world!".to_string(),
+            e: Some(vec![Some(true), None]),
+            f: Bar(Some(true)),
+        },
+        Foo {
+            a: 42,
+            b: None,
+            c: false,
+            d: "narrow".to_string(),
+            e: None,
+            f: Bar(None),
+        },
+    ];
+
+    let narrow_array = input
+        .into_iter()
+        .collect::<StructArray<Foo, false, ArrowBufferBuilder>>();
+
+    let record_batch = RecordBatch::from(narrow_array);
+    pretty::print_batches(&[record_batch.clone()]).unwrap();
+
+    let mut buffer = Vec::new();
+    let mut writer = ArrowWriter::try_new(&mut buffer, record_batch.schema(), None).unwrap();
+    writer.write(&record_batch).unwrap();
+    writer.close().unwrap();
+
+    let mut reader = ParquetRecordBatchReader::try_new(Bytes::from(buffer), 1024).unwrap();
+    let read = reader.next().unwrap().unwrap();
+    pretty::print_batches(&[read.clone()]).unwrap();
+    assert_eq!(record_batch, read);
+}

narrow-derive/Cargo.toml

@@ -12,6 +12,10 @@ license.workspace = true
 keywords.workspace = true
 categories.workspace = true
 
+[features]
+default = []
+arrow-rs = []
+
 [lib]
 proc-macro = true
 

narrow-derive/src/lib.rs

@@ -12,7 +12,8 @@ const CRATE: &str = "narrow";
 
 static NARROW: Lazy<String> = Lazy::new(|| match proc_macro_crate::crate_name(CRATE) {
     Ok(found) => match found {
-        FoundCrate::Itself => "crate".to_string(),
+        // Requires `extern crate self as narrow`
+        FoundCrate::Itself => CRATE.to_string(),
         FoundCrate::Name(name) => name,
     },
     _ => CRATE.to_string(),