Simplified code in flight integration tests

integration-testing/Cargo.toml

@@ -41,3 +41,4 @@ serde_json = { version = "1.0", features = ["preserve_order"] }
 tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread"] }
 tonic = "0.7.0"
 tracing-subscriber = { version = "0.3.1", optional = true }
+async-stream = { version = "0.3.2" }

integration-testing/README.md

@@ -1,30 +1,31 @@
-<!---
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
--->
-
-# Apache Arrow Rust Integration Testing
-
-See [Integration.rst](../../docs/source/format/Integration.rst) for an overview of integration testing.
-
-This crate contains the following binaries, which are invoked by Archery during integration testing with other Arrow implementations.
-
-| Binary | Purpose |
-|--------|---------|
-| arrow-file-to-stream | Converts an Arrow file to an Arrow stream |
-| arrow-stream-to-file | Converts an Arrow stream to an Arrow file |
-| arrow-json-integration-test | Converts between Arrow and JSON formats |
+# Integration tests
+
+This directory contains integration tests against official Arrow implementations.
+
+They are run as part of the CI pipeline, called by apache/arrow/dev/crossbow.
+
+The IPC files tested on the official pipeline are already tested on our own tests.
+
+## Flight tests
+
+To run the flight scenarios across this implementation, use
+
+```bash
+SCENARIO="auth:basic_proto"
+cargo run --bin flight-test-integration-server -- --port 3333 --scenario $SCENARIO &
+# wait for server to be up
+
+cargo run --bin flight-test-integration-client -- --host localhost --port 3333 --scenario $SCENARIO
+```
+
+to run an integration test against a file, use
+
+```bash
+FILE="../testing/arrow-testing/data/arrow-ipc-stream/integration/1.0.0-littleendian/generated_dictionary.json.gz"
+gzip -dc $FILE > generated.json
+
+cargo build --bin flight-test-integration-server
+cargo run --bin flight-test-integration-server -- --port 3333 &
+cargo run --bin flight-test-integration-client -- --host localhost --port 3333 --path generated.json
+# kill with `fg` and stop process
+```

integration-testing/src/bin/arrow-json-integration-test.rs

@@ -64,7 +64,7 @@ fn json_to_arrow(json_name: &str, arrow_name: &str, verbose: bool) -> Result<()>
         options,
     )?;
 
-    for b in json_file.batches {
+    for b in json_file.chunks {
         writer.write(&b, None)?;
     }
 
@@ -129,7 +129,7 @@ fn validate(arrow_name: &str, json_name: &str, verbose: bool) -> Result<()> {
         )));
     }
 
-    let json_batches = json_file.batches;
+    let json_batches = json_file.chunks;
 
     if verbose {
         eprintln!(