ARROW-11417: [Integration] Add integration tests for buffer compression

The tests use "gold" files generated by PyArrow 2.0.0 and are only enabled for C++. Closes #9408 from pitrou/ARROW-11417-compression-integration Authored-by: Antoine Pitrou <[email protected]> Signed-off-by: Antoine Pitrou <[email protected]>
apache · Feb 9, 2021 · 0a435f4 · 0a435f4
1 parent 319b46c
commit 0a435f4
Show file tree

Hide file tree

Showing 6 changed files with 49 additions and 37 deletions.
diff --git a/ci/scripts/integration_arrow.sh b/ci/scripts/integration_arrow.sh
@@ -22,11 +22,12 @@ set -ex
 arrow_dir=${1}
 source_dir=${1}/cpp
 build_dir=${2}/cpp
-gold_dir_0_14_1=$arrow_dir/testing/data/arrow-ipc-stream/integration/0.14.1
-gold_dir_0_17_1=$arrow_dir/testing/data/arrow-ipc-stream/integration/0.17.1
+
+gold_dir=$arrow_dir/testing/data/arrow-ipc-stream/integration
 
 pip install -e $arrow_dir/dev/archery
 
 archery integration --with-all --run-flight \
-    --gold-dirs=$gold_dir_0_14_1 \
-    --gold-dirs=$gold_dir_0_17_1 \
+    --gold-dirs=$gold_dir/0.14.1 \
+    --gold-dirs=$gold_dir/0.17.1 \
+    --gold-dirs=$gold_dir/2.0.0-compression \
diff --git a/dev/archery/archery/integration/runner.py b/dev/archery/archery/integration/runner.py
@@ -128,6 +128,11 @@ def _gold_tests(self, gold_dir):
                 skip = set()
             if name == 'union' and prefix == '0.17.1':
                 skip.add("Java")
+            if prefix == '2.0.0-compression':
+                skip.add("Go")
+                skip.add("Java")
+                skip.add("JS")
+                skip.add("Rust")
             yield datagen.File(name, None, None, skip=skip, path=out_path)
 
     def _run_test_cases(self, producer, consumer, case_runner,

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1147,9 +1147,16 @@ services:
     volumes: *conda-volumes
     environment:
       <<: *ccache
+      ARROW_BUILD_STATIC: "OFF"
+      ARROW_COMPUTE: "OFF"
+      ARROW_CSV: "OFF"
+      ARROW_FLIGHT: "ON"
+      ARROW_GANDIVA: "OFF"
+      ARROW_PARQUET: "OFF"
+      ARROW_S3: "OFF"
       # tell archery where the arrow binaries are located
       ARROW_CPP_EXE_PATH: /build/cpp/debug
-    # Running integration tests serially until ARROW-8176 resolved
+      CMAKE_UNITY_BUILD: "ON"
     command:
       ["/arrow/ci/scripts/rust_build.sh /arrow /build &&
         /arrow/ci/scripts/cpp_build.sh /arrow /build &&

diff --git a/js/package-lock.json b/js/package-lock.json
diff --git a/rust/arrow-flight/src/arrow.flight.protocol.rs b/rust/arrow-flight/src/arrow.flight.protocol.rs
@@ -499,9 +499,8 @@ pub mod flight_service_server {
     #[async_trait]
     pub trait FlightService: Send + Sync + 'static {
         #[doc = "Server streaming response type for the Handshake method."]
-        type HandshakeStream: Stream<
-            Item = Result<super::HandshakeResponse, tonic::Status>,
-        > + Send
+        type HandshakeStream: Stream<Item = Result<super::HandshakeResponse, tonic::Status>>
+            + Send
             + Sync
             + 'static;
         #[doc = ""]

diff --git a/testing b/testing
+ −		data/arrow-ipc-stream/integration/2.0.0-compression/generated_lz4.arrow_file
+ −		data/arrow-ipc-stream/integration/2.0.0-compression/generated_lz4.json.gz
+ −		data/arrow-ipc-stream/integration/2.0.0-compression/generated_lz4.stream
+ −		data/arrow-ipc-stream/integration/2.0.0-compression/generated_zstd.arrow_file
+ −		data/arrow-ipc-stream/integration/2.0.0-compression/generated_zstd.json.gz
+ −		data/arrow-ipc-stream/integration/2.0.0-compression/generated_zstd.stream
+7 −0		data/arrow-ipc-tensor-stream/README.md
+ −		data/arrow-ipc-tensor-stream/clusterfuzz-testcase-minimized-arrow-ipc-tensor-stream-fuzz-6232257264222208
+ −		data/arrow-ipc-tensor-stream/clusterfuzz-testcase-minimized-arrow-ipc-tensor-stream-fuzz-6583463383793664
+ −		data/parquet/fuzzing/clusterfuzz-testcase-parquet-arrow-fuzz-6015129204424704