diff --git a/.github/workflows/integration-ffi.yml b/.github/workflows/integration-ffi.yml index 7f7b4ce3d35..f09f0955880 100644 --- a/.github/workflows/integration-ffi.yml +++ b/.github/workflows/integration-ffi.yml @@ -25,7 +25,7 @@ jobs: key: ${{ runner.os }}-amd64-target-maturin-cache - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.10" - name: Install Python dependencies run: python -m pip install --upgrade pip setuptools wheel - name: Run tests @@ -38,6 +38,6 @@ jobs: python -m venv venv source venv/bin/activate - pip install maturin==0.10.2 toml==0.10.1 pyarrow==6.0.0 + pip install maturin==0.12.6 toml==0.10.2 pyarrow==6 maturin develop python -m unittest discover tests diff --git a/.github/workflows/integration-ipc.yml b/.github/workflows/integration-ipc.yml index acea8b0a409..11aa79bed50 100644 --- a/.github/workflows/integration-ipc.yml +++ b/.github/workflows/integration-ipc.yml @@ -31,7 +31,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: "3.10" - name: Setup Archery run: pip install -e dev/archery[docker] - name: Execute Docker Build diff --git a/.github/workflows/integration-parquet.yml b/.github/workflows/integration-parquet.yml index 208098917b8..9e253a7491d 100644 --- a/.github/workflows/integration-parquet.yml +++ b/.github/workflows/integration-parquet.yml @@ -27,7 +27,7 @@ jobs: key: ${{ runner.os }}-amd64-target-parquet-cache - uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: "3.10" - name: Build run: | export CARGO_HOME="/home/runner/.cargo" @@ -46,7 +46,7 @@ jobs: python -m venv venv source venv/bin/activate pip install --upgrade pip - pip install pyarrow pyspark + pip install pyarrow==6 pyspark==3 python main.py # test against spark python main_spark.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dcff300eeba..324a4f82524 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,9 +18,11 @@ jobs: run: | apt update && apt install python3-pip python3-venv -y -q python3 -m venv venv - venv/bin/pip install pip --upgrade - venv/bin/pip install pyarrow==3 - venv/bin/python parquet_integration/write_parquet.py + source venv/bin/activate + pip install pip --upgrade + pip install pyarrow==6 + python parquet_integration/write_parquet.py + deactivate - name: Run run: cargo test --features full @@ -122,9 +124,11 @@ jobs: run: | apt update && apt install python3-pip python3-venv -y -q python3 -m venv venv - venv/bin/pip install pip --upgrade - venv/bin/pip install pyarrow==3 - venv/bin/python parquet_integration/write_parquet.py + source venv/bin/activate + pip install pip --upgrade + pip install pyarrow==6 + python parquet_integration/write_parquet.py + deactivate - uses: Swatinem/rust-cache@v1 - name: Install tarpaulin run: cargo install cargo-tarpaulin diff --git a/arrow-pyarrow-integration-testing/README.md b/arrow-pyarrow-integration-testing/README.md index 7e78aa9ec70..2d1bdca4b94 100644 --- a/arrow-pyarrow-integration-testing/README.md +++ b/arrow-pyarrow-integration-testing/README.md @@ -22,9 +22,10 @@ This is a Rust crate that tests compatibility between Rust's Arrow implementation and PyArrow. Note that this crate uses two languages and an external ABI: -* `Rust` -* `Python` -* C ABI privately exposed by `Pyarrow`. + +- `Rust` +- `Python` +- C ABI privately exposed by `Pyarrow`. ## Basic idea @@ -35,17 +36,19 @@ we can use pyarrow's interface to move pointers from and to Rust. ## Relevant literature -* [Arrow's CDataInterface](https://arrow.apache.org/docs/format/CDataInterface.html) -* [Rust's FFI](https://doc.rust-lang.org/nomicon/ffi.html) -* [Pyarrow private binds](https://github.com/apache/arrow/blob/ae1d24efcc3f1ac2a876d8d9f544a34eb04ae874/python/pyarrow/array.pxi#L1226) -* [PyO3](https://docs.rs/pyo3/0.12.1/pyo3/index.html) +- [Arrow's CDataInterface](https://arrow.apache.org/docs/format/CDataInterface.html) +- [Rust's FFI](https://doc.rust-lang.org/nomicon/ffi.html) +- [Pyarrow private binds](https://github.com/apache/arrow/blob/ae1d24efcc3f1ac2a876d8d9f544a34eb04ae874/python/pyarrow/array.pxi#L1226) +- [PyO3](https://docs.rs/pyo3/0.12.1/pyo3/index.html) ## How to develop ```bash # prepare development environment (used to build wheel / install in development) python -m venv venv -venv/bin/pip install maturin==0.8.2 toml==0.10.1 pyarrow==1.0.0 +source venv/bin/activate +pip install maturin==0.12.6 toml==0.10.2 pyarrow==6 +deactivate ``` Whenever rust code changes (your changes or via git pull):