Merge branch 'main' into bitwise-ops

.cargo/audit.toml

@@ -0,0 +1,13 @@
+[advisories]
+ignore = [
+    # title: Potential segfault in the time crate
+    # This can be ignored because it only affects users that use the feature flag "clock" of "chrono",
+    # which we do not. Specifically: 
+    # * the call of "localtime_r" [is unsound](https://github.com/chronotope/chrono/issues/602#issuecomment-940445390)
+    # * that call [is part of the module "sys"](https://docs.rs/chrono/0.4.19/src/chrono/sys/unix.rs.html#84)
+    # * "sys" is only available on feature "clock": https://docs.rs/chrono/0.4.19/src/chrono/lib.rs.html#456
+    # 
+    # Therefore, this advisory does not affect us.
+    "RUSTSEC-2020-0071",
+    "RUSTSEC-2020-0159", # same as previous
+]

.github/workflows/test.yml

@@ -74,7 +74,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-07-09
+          toolchain: nightly-2021-10-24
           override: true
       - uses: Swatinem/rust-cache@v1
       - name: Install Miri
@@ -93,7 +93,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-07-09
+          toolchain: nightly-2021-10-24
           override: true
       - uses: Swatinem/rust-cache@v1
       - name: Install Miri
@@ -112,6 +112,8 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
+      - name: Install Rust
+        run: rustup update stable
       - name: Setup parquet files
         run: |
           apt update && apt install python3-pip python3-venv -y -q

.github_changelog_generator

@@ -1,5 +1,5 @@
-since-tag=v0.6.1
-future-release=v0.6.2
+since-tag=v0.6.2
+future-release=v0.7.0
 pr-wo-labels=false
 exclude-labels=no-changelog,question
 add-sections={"features":{"prefix":"**Enhancements:**","labels":["enhancement"]}, "documentation":{"prefix":"**Documentation updates:**","labels":["documentation"]}, "testing":{"prefix":"**Testing updates:**","labels":["testing"]}}

CHANGELOG.md

@@ -1,5 +1,50 @@
 # Changelog
 
+## [v0.7.0](https://github.com/jorgecarleitao/arrow2/tree/v0.7.0) (2021-10-29)
+
+[Full Changelog](https://github.com/jorgecarleitao/arrow2/compare/v0.6.2...v0.7.0)
+
+**Breaking changes:**
+
+- Simplified reading parquet [\#532](https://github.com/jorgecarleitao/arrow2/pull/532) ([jorgecarleitao](https://github.com/jorgecarleitao))
+- Change IPC `FileReader` to own the underlying reader [\#518](https://github.com/jorgecarleitao/arrow2/pull/518) ([blakesmith](https://github.com/blakesmith))
+- Migrate to `arrow_format` crate [\#517](https://github.com/jorgecarleitao/arrow2/pull/517) ([jorgecarleitao](https://github.com/jorgecarleitao))
+
+**New features:**
+
+- Added read of 2-level nested lists from parquet [\#548](https://github.com/jorgecarleitao/arrow2/pull/548) ([jorgecarleitao](https://github.com/jorgecarleitao))
+- add dictionary serialization for csv-writer [\#515](https://github.com/jorgecarleitao/arrow2/pull/515) ([ritchie46](https://github.com/ritchie46))
+- Added `checked_negate` and `wrapping_negate` for `PrimitiveArray` [\#506](https://github.com/jorgecarleitao/arrow2/pull/506) ([yjhmelody](https://github.com/yjhmelody))
+
+**Fixed bugs:**
+
+- Fixed error in reading fixed len binary from parquet [\#549](https://github.com/jorgecarleitao/arrow2/pull/549) ([jorgecarleitao](https://github.com/jorgecarleitao))
+- Fixed ffi of sliced arrays [\#540](https://github.com/jorgecarleitao/arrow2/pull/540) ([jorgecarleitao](https://github.com/jorgecarleitao))
+- Fixed s3 example [\#536](https://github.com/jorgecarleitao/arrow2/pull/536) ([jorgecarleitao](https://github.com/jorgecarleitao))
+- Fixed error in writing compressed parquet dict pages [\#523](https://github.com/jorgecarleitao/arrow2/pull/523) ([jorgecarleitao](https://github.com/jorgecarleitao))
+- Validity taken into account when writing `StructArray` to json [\#511](https://github.com/jorgecarleitao/arrow2/pull/511) ([VasanthakumarV](https://github.com/VasanthakumarV))
+
+**Enhancements:**
+
+- Bumped Prost and Tonic [\#550](https://github.com/jorgecarleitao/arrow2/pull/550) ([PsiACE](https://github.com/PsiACE))
+- Speedup scalar boolean operations [\#546](https://github.com/jorgecarleitao/arrow2/pull/546) ([Dandandan](https://github.com/Dandandan))
+- Added fast path for validating ASCII text \(~1.12-1.89x improvement on reading ASCII parquet data\) [\#542](https://github.com/jorgecarleitao/arrow2/pull/542) ([Dandandan](https://github.com/Dandandan))
+- Exposed missing APIs to write parquet in parallel [\#539](https://github.com/jorgecarleitao/arrow2/pull/539) ([jorgecarleitao](https://github.com/jorgecarleitao))
+- improve utf8 init validity [\#530](https://github.com/jorgecarleitao/arrow2/pull/530) ([ritchie46](https://github.com/ritchie46))
+- export missing `BinaryValueIter` [\#526](https://github.com/jorgecarleitao/arrow2/pull/526) ([yjhmelody](https://github.com/yjhmelody))
+
+**Documentation updates:**
+
+- Added more IPC documentation [\#534](https://github.com/jorgecarleitao/arrow2/pull/534) ([HagaiHargil](https://github.com/HagaiHargil))
+- Fixed clippy and fmt [\#521](https://github.com/jorgecarleitao/arrow2/pull/521) ([ritchie46](https://github.com/ritchie46))
+
+**Testing updates:**
+
+- Added more tests for `utf8` [\#543](https://github.com/jorgecarleitao/arrow2/pull/543) ([jorgecarleitao](https://github.com/jorgecarleitao))
+- Ignored RUSTSEC-2020-0071 and RUSTSEC-2020-0159 [\#537](https://github.com/jorgecarleitao/arrow2/pull/537) ([jorgecarleitao](https://github.com/jorgecarleitao))
+- Improved parquet read benches [\#533](https://github.com/jorgecarleitao/arrow2/pull/533) ([jorgecarleitao](https://github.com/jorgecarleitao))
+- Added fmt and clippy checks to CI. [\#522](https://github.com/jorgecarleitao/arrow2/pull/522) ([xudong963](https://github.com/xudong963))
+
 ## [v0.6.2](https://github.com/jorgecarleitao/arrow2/tree/v0.6.2) (2021-10-09)
 
 [Full Changelog](https://github.com/jorgecarleitao/arrow2/compare/v0.6.1...v0.6.2)

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "arrow2"
-version = "0.6.2"
+version = "0.7.0"
 license = "Apache-2.0"
 description = "Unofficial implementation of Apache Arrow spec in safe Rust"
 homepage = "https://github.com/jorgecarleitao/arrow2"
@@ -17,7 +17,7 @@ bench = false
 [dependencies]
 num-traits = "0.2"
 chrono = { version = "0.4", default_features = false, features = ["std"] }
-chrono-tz = { version = "0.5", optional = true }
+chrono-tz = { version = "0.6", optional = true }
 # To efficiently cast numbers to strings
 lexical-core = { version = "0.8", optional = true }
 # We need to Hash values before sending them to an hasher. This
@@ -30,6 +30,7 @@ csv = { version = "^1.1", optional = true }
 regex = { version = "^1.3", optional = true }
 lazy_static = { version = "^1.4", optional = true }
 streaming-iterator = { version = "0.1", optional = true }
+fallible-streaming-iterator = { version = "0.1", optional = true }
 
 serde = { version = "^1.0", features = ["rc"], optional = true }
 serde_derive = { version = "^1.0", optional = true }
@@ -39,7 +40,8 @@ indexmap = { version = "^1.6", optional = true }
 # used to print columns in a nice columnar format
 comfy-table = { version = "4.0", optional = true, default-features = false }
 
-flatbuffers = { version = "=2.0.0", optional = true }
+arrow-format = { version = "0.3.0",  optional = true, features = ["ipc"] }
+
 hex = { version = "^0.4", optional = true }
 
 # for IPC compression
@@ -60,10 +62,12 @@ futures = { version = "0.3", optional = true }
 # for faster hashing
 ahash = { version = "0.7", optional = true }
 
-parquet2 = { version = "0.5.2", optional = true, default_features = false, features = ["stream"] }
+parquet2 = { version = "0.6", optional = true, default_features = false, features = ["stream"] }
 
 avro-rs = { version = "0.13", optional = true, default_features = false }
 
+libflate = { version = "1.1.1", optional = true }
+
 # for division/remainder optimization at runtime
 strength_reduce = { version = "0.2", optional = true }
 
@@ -88,6 +92,7 @@ full = [
     "io_csv",
     "io_json",
     "io_ipc",
+    "io_flight",
     "io_ipc_compression",
     "io_json_integration",
     "io_print",
@@ -105,16 +110,17 @@ io_csv = ["io_csv_read", "io_csv_write"]
 io_csv_read = ["csv", "lexical-core"]
 io_csv_write = ["csv", "streaming-iterator", "lexical-core"]
 io_json = ["serde", "serde_json", "indexmap"]
-io_ipc = ["flatbuffers"]
+io_ipc = ["arrow-format"]
 io_ipc_compression = ["lz4", "zstd"]
+io_flight = ["io_ipc", "arrow-format/flight-data"]
 io_parquet_compression = [
     "parquet2/zstd",
     "parquet2/snappy",
     "parquet2/gzip",
     "parquet2/lz4",
     "parquet2/brotli",
 ]
-io_avro = ["avro-rs", "streaming-iterator", "serde_json"]
+io_avro = ["avro-rs", "fallible-streaming-iterator", "serde_json", "libflate"]
 # io_json: its dependencies + error handling
 # serde_derive: there is some derive around
 io_json_integration = ["io_json", "serde_derive", "hex"]
@@ -143,6 +149,8 @@ skip_feature_sets = [
     ["io_csv_write"],
     ["io_avro"],
     ["io_json"],
+    ["io_flight"],
+    ["io_ipc"],
     ["io_parquet"],
     ["io_json_integration"],
     # this does not change the public API
@@ -238,6 +246,10 @@ harness = false
 name = "iter_list"
 harness = false
 
+[[bench]]
+name = "avro_read"
+harness = false
+
 [[bench]]
 name = "bitwise"
 harness = false