Write categorical directly to csv wo/ cast

.github/deploy_manylinux.sh

@@ -7,7 +7,7 @@ ls -la
 rm py-polars/README.md
 cp README.md py-polars/README.md
 cd py-polars
-rustup override set nightly-2021-10-03
+rustup override set nightly-2021-10-09
 export RUSTFLAGS='-C target-feature=+fxsr,+sse,+sse2,+sse3,+ssse3,+sse4.1,+popcnt'
 maturin publish \
   --skip-existing \

.github/workflows/benchmark.yaml

@@ -10,7 +10,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-10-03
+          toolchain: nightly-2021-10-09
           override: true
           components: rustfmt, clippy
       - name: Set up Python
@@ -31,7 +31,7 @@ jobs:
           pip install -r py-polars/build.requirements.txt
           pip install --only-binary all pyarrow
           cd py-polars
-          rustup override set nightly-2021-10-03 && RUSTFLAGS="-C embed-bitcode" maturin develop --rustc-extra-args="-C codegen-units=8 -C lto=thin -C target-cpu=native" --release
+          rustup override set nightly-2021-10-09 && RUSTFLAGS="-C embed-bitcode" maturin develop --rustc-extra-args="-C codegen-units=8 -C lto=thin -C target-cpu=native" --release
           cd tests/db-benchmark
           Rscript -e 'install.packages("data.table", repos="https://Rdatatable.github.io/data.table")'
           Rscript groupby-datagen.R 1e7 1e2 5 0

.github/workflows/build-test.yaml

@@ -12,7 +12,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-10-03
+          toolchain: nightly-2021-10-09
           override: true
           components: rustfmt, clippy, miri
       - name: Cache Cargo
@@ -70,7 +70,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-10-03
+          toolchain: nightly-2021-10-09
           override: true
           components: rustfmt, clippy
       - name: Set up Python
@@ -93,7 +93,7 @@ jobs:
       - name: Run tests
         run: |
           export RUSTFLAGS="-C debuginfo=0"
-          cd py-polars && rustup override set nightly-2021-10-03 && make build-and-test
+          cd py-polars && rustup override set nightly-2021-10-09 && make build-and-test
           cargo clippy
       # test if we can import polars without any requirements
       - name: Import polars

.github/workflows/create-py-release-test.yaml

@@ -17,7 +17,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-10-03
+          toolchain: nightly-2021-10-09
           override: true
           components: rustfmt, clippy
       - name: Set up Python
@@ -31,7 +31,7 @@ jobs:
       - name: Run tests
         run: |
           cd py-polars
-          rustup override set nightly-2021-10-03
+          rustup override set nightly-2021-10-09
           make build-and-test
           rm wheels/*
 
@@ -48,7 +48,7 @@ jobs:
         - name: Install latest Rust nightly
           uses: actions-rs/toolchain@v1
           with:
-            toolchain: nightly-2021-10-03
+            toolchain: nightly-2021-10-09
             override: true
             components: rustfmt, clippy
         - name: Set up Python
@@ -67,7 +67,7 @@ jobs:
             rm py-polars/README.md
             cp README.md py-polars/README.md
             cd py-polars
-            rustup override set nightly-2021-10-03
+            rustup override set nightly-2021-10-09
             export RUSTFLAGS='-C target-feature=+fxsr,+sse,+sse2,+sse3'
             maturin publish \
             --skip-existing \

.github/workflows/create-py-universal2-release.yaml

@@ -19,7 +19,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-10-03
+          toolchain: nightly-2021-10-09
           override: true
           components: rustfmt, clippy
       - name: Setup universal2 targets for Rust
@@ -35,7 +35,7 @@ jobs:
           rm py-polars/README.md
           cp README.md py-polars/README.md
           cd py-polars
-          rustup override set nightly-2021-10-03
+          rustup override set nightly-2021-10-09
       - name: maturin publish
         uses: messense/maturin-action@v1
         env:

.github/workflows/docs.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-10-03
+          toolchain: nightly-2021-10-09
           override: true
           components: rustfmt, clippy
       - name: Set up Python

.github/workflows/test-windows.yaml

@@ -10,7 +10,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-10-03
+          toolchain: nightly-2021-10-09
           override: true
       - name: Run tests
         run: |
@@ -24,7 +24,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-10-03
+          toolchain: nightly-2021-10-09
           override: true
           components: rustfmt, clippy
       - name: Set up Python
@@ -39,7 +39,7 @@ jobs:
         shell: bash
         run: |
           export RUSTFLAGS="-C debuginfo=0"
-          cd py-polars && rustup override set nightly-2021-10-03 && make build-and-test
+          cd py-polars && rustup override set nightly-2021-10-09 && make build-and-test
           cargo clippy
       # test if we can import polars without any requirements
       - name: Import polars

polars/polars-arrow/Cargo.toml

@@ -9,8 +9,8 @@ description = "Arrow interfaces for Polars DataFrame library"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-#arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "194a95d52978b60859a3b2846f8e86a89bc15990", default-features = false }
-arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch="dev", default-features = false }
+arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "25ddb0d5fc2dfea80f3ac52eba5c26c294743da8", default-features = false }
+#arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch="dev", default-features = false }
 #arrow = { package = "arrow2", version = "0.5.3", default-features=false}
 thiserror = "^1.0"
 num = "^0.4"

polars/polars-core/Cargo.toml

@@ -123,8 +123,8 @@ docs-selection = [
 ]
 
 [dependencies]
-#arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "194a95d52978b60859a3b2846f8e86a89bc15990", default-features = false, features=["compute"] }
-arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", default-features = false, features=["compute"], branch="dev" }
+arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "25ddb0d5fc2dfea80f3ac52eba5c26c294743da8", default-features = false, features=["compute"] }
+#arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", default-features = false, features=["compute"], branch="dev" }
 #arrow = { package = "arrow2", version="0.5.3", default-features = false, features=["compute"]}
 polars-arrow = {version = "0.16.0", path = "../polars-arrow"}
 thiserror = "1.0"

polars/polars-io/Cargo.toml

@@ -28,8 +28,8 @@ temporal = ["polars-core/dtype-date", "polars-core/dtype-datetime"]
 private = []
 
 [dependencies]
-#arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "194a95d52978b60859a3b2846f8e86a89bc15990", default-features = false }
-arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", default-features = false, branch="dev"}
+arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "25ddb0d5fc2dfea80f3ac52eba5c26c294743da8", default-features = false }
+#arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", default-features = false, branch="dev"}
 #arrow = { package = "arrow2", version="0.5.3", --default-features=false }
 polars-core = {version = "0.16.0", path = "../polars-core", features = ["private"], default-features=false}
 polars-arrow = {version = "0.16.0", path = "../polars-arrow"}

polars/polars-io/src/csv.rs

@@ -81,21 +81,6 @@ where
 
     fn finish(self, df: &DataFrame) -> Result<()> {
         let mut writer = self.writer_builder.from_writer(self.buffer);
-
-        // temp coerce cat to utf8 until supported in csv writer
-        let columns = df
-            .get_columns()
-            .iter()
-            .map(|s| {
-                if let DataType::Categorical = s.dtype() {
-                    s.cast(&DataType::Utf8).unwrap()
-                } else {
-                    s.clone()
-                }
-            })
-            .collect();
-        let df = DataFrame::new_no_checks(columns);
-
         let iter = df.iter_record_batches();
         write::write_header(&mut writer, &df.schema().to_arrow())?;
         for batch in iter {

polars/polars-io/src/ipc.rs

@@ -62,7 +62,7 @@ pub struct IpcReader<R> {
     rechunk: bool,
 }
 
-impl<'a, R> ArrowReader for read::FileReader<'a, R>
+impl<R> ArrowReader for read::FileReader<R>
 where
     R: Read + Seek,
 {