diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml new file mode 100644 index 00000000000..b4d2c8b13c6 --- /dev/null +++ b/.github/workflows/cancel.yml @@ -0,0 +1,13 @@ +name: Cancel +on: + workflow_run: + workflows: ["test"] + types: + - requested +jobs: + cancel: + runs-on: ubuntu-latest + steps: + - uses: styfle/cancel-workflow-action@0.8.0 + with: + workflow_id: ${{ github.event.workflow.id }} diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000000..69d448cd7ca --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,34 @@ +name: Coverage + +on: [pull_request, push] + +jobs: + coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true # needed to test IPC, which are located in a submodule + - name: Install Rust + run: rustup toolchain install stable --component llvm-tools-preview + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - name: Setup parquet files + run: | + apt update && apt install python3-pip python3-venv -y -q + python3 -m venv venv + source venv/bin/activate + pip install pip --upgrade + pip install pyarrow==6 pyorc + python parquet_integration/write_parquet.py + python tests/it/io/orc/write.py + deactivate + - uses: Swatinem/rust-cache@v1 + - name: Generate code coverage + run: cargo llvm-cov --features full --lcov --output-path lcov.info + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + files: lcov.info + fail_ci_if_error: true diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 746807835a3..2a1d92de9a1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,7 +16,7 @@ jobs: - run: mdbook build guide - name: Deploy - if: github.ref == 'refs/heads/docs' || startsWith(github.ref, 'refs/tags/v') + if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') uses: peaceiris/actions-gh-pages@v3.7.3 with: personal_token: ${{ secrets.GITHUB_TOKEN }} @@ -45,7 +45,7 @@ jobs: RUSTDOCFLAGS: --cfg docsrs - name: Deploy - if: github.ref == 'refs/heads/docs' || startsWith(github.ref, 'refs/tags/v') + if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') uses: peaceiris/actions-gh-pages@v3.7.3 with: personal_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/integration-ffi.yml b/.github/workflows/integration-ffi.yml new file mode 100644 index 00000000000..f09f0955880 --- /dev/null +++ b/.github/workflows/integration-ffi.yml @@ -0,0 +1,43 @@ +name: Integration FFI + +on: [push, pull_request] + +jobs: + docker: + name: Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Setup Rust toolchain + run: | + rustup toolchain install stable + rustup default stable + rustup component add rustfmt clippy + - name: Cache Cargo + uses: actions/cache@v2 + with: + path: /home/runner/.cargo + key: cargo-maturin-cache- + - name: Cache Rust dependencies + uses: actions/cache@v2 + with: + path: /home/runner/target + key: ${{ runner.os }}-amd64-target-maturin-cache + - uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Install Python dependencies + run: python -m pip install --upgrade pip setuptools wheel + - name: Run tests + run: | + export CARGO_HOME="/home/runner/.cargo" + export CARGO_TARGET_DIR="/home/runner/target" + + cd arrow-pyarrow-integration-testing + + python -m venv venv + source venv/bin/activate + + pip install maturin==0.12.6 toml==0.10.2 pyarrow==6 + maturin develop + python -m unittest discover tests diff --git a/.github/workflows/integration-ipc.yml b/.github/workflows/integration-ipc.yml new file mode 100644 index 00000000000..11aa79bed50 --- /dev/null +++ b/.github/workflows/integration-ipc.yml @@ -0,0 +1,38 @@ +name: Integration IPC / Flight + +on: [push, pull_request] + +jobs: + docker: + name: Test + runs-on: ubuntu-latest + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + repository: apache/arrow + submodules: true + fetch-depth: 0 + # this is temporary: once rust is removed from `apache/arrow`, we are good to go. + - name: Remove Rust from arrow + run: rm -rf rust/ + - name: Checkout Arrow Rust + uses: actions/checkout@v2 + with: + path: rust + fetch-depth: 0 + # Arrow uses cargo build -p arrow-integration-testing + # which is incompatible with this. Let's monkey patch it + - name: Fix compilation + run: cp rust/integration-testing/rust_build.sh ci/scripts/rust_build.sh + # unskip many of the tests + - name: Test more cases + run: git apply rust/integration-testing/unskip.patch + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: "3.10" + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: archery docker run -e ARCHERY_INTEGRATION_WITH_RUST=1 conda-integration diff --git a/.github/workflows/integration-odbc.yml b/.github/workflows/integration-odbc.yml new file mode 100644 index 00000000000..2ee3bf096f2 --- /dev/null +++ b/.github/workflows/integration-odbc.yml @@ -0,0 +1,40 @@ +name: Integration ODBC + +on: [push, pull_request] + +env: + CARGO_TERM_COLOR: always + +jobs: + linux: + name: Test + runs-on: ubuntu-latest + + services: + sqlserver: + image: mcr.microsoft.com/mssql/server:2017-latest-ubuntu + ports: + - 1433:1433 + env: + ACCEPT_EULA: Y + SA_PASSWORD: My@Test@Password1 + + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install ODBC Drivers + run: | + curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - + curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list > /etc/apt/sources.list.d/mssql-release.list + apt-get update + ACCEPT_EULA=Y apt-get install -y msodbcsql17 + ln -s /opt/microsoft/msodbcsql17/lib64/libmsodbcsql-17.*.so.* /opt/microsoft/msodbcsql17/lib64/libmsodbcsql-17.so + shell: sudo bash {0} + - name: Setup Rust toolchain + run: | + rustup toolchain install stable + rustup default stable + rustup component add rustfmt clippy + - uses: Swatinem/rust-cache@v1 + - name: Test + run: cd arrow-odbc-integration-testing && cargo test diff --git a/.github/workflows/integration-parquet.yml b/.github/workflows/integration-parquet.yml new file mode 100644 index 00000000000..53a76d5b52f --- /dev/null +++ b/.github/workflows/integration-parquet.yml @@ -0,0 +1,52 @@ +name: Integration Parquet + +on: [push, pull_request] + +jobs: + docker: + name: Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Setup Rust toolchain + run: | + rustup toolchain install stable + rustup default stable + rustup component add rustfmt clippy + - name: Cache Cargo + uses: actions/cache@v2 + with: + path: /home/runner/.cargo + key: cargo-parquet-cache- + - name: Cache Rust dependencies + uses: actions/cache@v2 + with: + path: /home/runner/target + key: ${{ runner.os }}-amd64-target-parquet-cache + - uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Build + run: | + export CARGO_HOME="/home/runner/.cargo" + export CARGO_TARGET_DIR="/home/runner/target" + + cd arrow-parquet-integration-testing + + cargo build + - name: Run + run: | + export CARGO_HOME="/home/runner/.cargo" + export CARGO_TARGET_DIR="/home/runner/target" + + cd arrow-parquet-integration-testing + + python -m venv venv + source venv/bin/activate + pip install --upgrade pip + pip install pyarrow==8 pyspark==3 + python main.py + # test against spark + python main_spark.py diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 00000000000..a1d19e6f8c6 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,10 @@ +name: Security audit +on: [push, pull_request] +jobs: + security_audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions-rs/audit-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000000..74ebe22eed5 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,191 @@ +name: Check and test + +on: [push, pull_request] + +jobs: + # test the crate + ubuntu-latest: + name: Test full + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true # needed to test IPC, which are located in a submodule + - name: Install Rust + run: rustup update stable + - name: Setup parquet files + run: | + apt update && apt install python3-pip python3-venv -y -q + python3 -m venv venv + source venv/bin/activate + pip install pip --upgrade + pip install pyarrow==6 pyorc + python parquet_integration/write_parquet.py + python tests/it/io/orc/write.py + deactivate + - uses: Swatinem/rust-cache@v1 + - name: Run + run: cargo test --features full + + windows-and-macos: + name: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + with: + submodules: true # needed to test IPC, which are located in a submodule + - name: Install Rust + run: rustup update stable + - uses: Swatinem/rust-cache@v1 + - name: Run + shell: bash + run: | + cargo check --features full + cargo test --tests + + clippy: + name: Clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install Rust + run: rustup update stable + - uses: Swatinem/rust-cache@v1 + - name: Install clippy + run: rustup component add clippy + - name: "clippy --all" + run: cargo clippy --all --features=full --tests -- -D warnings + + fmt: + name: fmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install Rust + run: rustup update stable + - uses: Swatinem/rust-cache@v1 + - name: Install rustfmt + run: rustup component add rustfmt + - name: Run + run: cargo fmt --all -- --check + + miri-checks: + name: Miri + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: nightly-2022-07-12 + override: true + - uses: Swatinem/rust-cache@v1 + with: + key: key1 + - name: Install Miri + run: | + rustup component add miri + cargo miri setup + + - name: Run + # --skip io: miri can't handle opening of files, so we skip those + run: cargo miri test --tests --features compute,chrono-tz + + miri-checks-io: + name: MIRI on IO IPC + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true # needed to test IPC, which are located in a submodule + - uses: actions-rs/toolchain@v1 + with: + toolchain: nightly-2022-07-12 + override: true + - uses: Swatinem/rust-cache@v1 + with: + key: key1 + - name: Install Miri + run: | + rustup component add miri + cargo miri setup + - name: Run + run: MIRIFLAGS="-Zmiri-disable-isolation" cargo miri test --tests --features io_ipc,io_json_integration io::ipc::write::write_sliced_list + + miri-checks-mmap: + name: MIRI on IO IPC mmaping + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: nightly-2022-07-12 + override: true + - uses: Swatinem/rust-cache@v1 + with: + key: key1 + - name: Install Miri + run: | + rustup component add miri + cargo miri setup + - name: Run + run: cargo miri test --tests --features io_ipc io::ipc::mmap + + feature-compilation: + name: Feature coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install Rust + run: rustup update stable + - name: Setup all features + run: cargo install cargo-all-features + - uses: Swatinem/rust-cache@v1 + - name: Run + run: cargo check-all-features + - name: Bench Check + run: cargo bench --no-run --features full,benchmarks + + cross: + name: cross + runs-on: ubuntu-latest + strategy: + matrix: + target: + - wasm32-unknown-unknown + - i686-unknown-linux-gnu + - powerpc-unknown-linux-gnu + - powerpc64-unknown-linux-gnu + - mips-unknown-linux-gnu + - arm-linux-androideabi + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions-rs/toolchain@v1 + with: + toolchain: nightly-2022-03-16 + target: ${{ matrix.target }} + override: true + - uses: Swatinem/rust-cache@v1 + - uses: actions-rs/cargo@v1 + with: + use-cross: true + command: check + args: --features=compute_merge_sort,io_ipc,io_csv,io_print,io_json,io_parquet --target ${{ matrix.target }} + + linux-simd-test: + name: SIMD + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: nightly-2022-07-24 + override: true + - uses: Swatinem/rust-cache@v1 + - name: Run + # no need to run over all features: simd only affects the core + run: cargo test --tests --no-default-features --features compute,simd