diff --git a/src/io/parquet/read/mod.rs b/src/io/parquet/read/mod.rs index a9e64c81d5f..a036faf4fc3 100644 --- a/src/io/parquet/read/mod.rs +++ b/src/io/parquet/read/mod.rs @@ -1,12 +1,13 @@ use std::io::{Read, Seek}; -use futures::Stream; +use futures::{AsyncRead, AsyncSeek, Stream}; pub use parquet2::{ error::ParquetError, metadata::{ColumnChunkMetaData, ColumnDescriptor, RowGroupMetaData}, page::{CompressedDataPage, DataPage, DataPageHeader}, read::{ - decompress, get_page_iterator as _get_page_iterator, read_metadata as _read_metadata, + decompress, get_page_iterator as _get_page_iterator, get_page_stream as _get_page_stream, + read_metadata as _read_metadata, read_metadata_async as _read_metadata_async, streaming_iterator, Decompressor, PageIterator, StreamingIterator, }, schema::types::{ @@ -48,11 +49,29 @@ pub fn get_page_iterator<'b, RR: Read + Seek>( )?) } -/// Reads parquets' metadata. +/// Creates a new iterator of compressed pages. +pub async fn get_page_stream<'a, RR: AsyncRead + Unpin + Send + AsyncSeek>( + metadata: &'a FileMetaData, + row_group: usize, + column: usize, + reader: &'a mut RR, + buffer: Vec, +) -> Result> + 'a> { + Ok(_get_page_stream(metadata, row_group, column, reader, buffer).await?) +} + +/// Reads parquets' metadata syncronously. pub fn read_metadata(reader: &mut R) -> Result { Ok(_read_metadata(reader)?) } +/// Reads parquets' metadata asynchronously. +pub async fn read_metadata_async( + reader: &mut R, +) -> Result { + Ok(_read_metadata_async(reader).await?) +} + pub fn page_iter_to_array< I: StreamingIterator>, >( diff --git a/src/io/parquet/write/utf8/nested.rs b/src/io/parquet/write/utf8/nested.rs index 65e7d3d13bf..d9b7dafbd0e 100644 --- a/src/io/parquet/write/utf8/nested.rs +++ b/src/io/parquet/write/utf8/nested.rs @@ -1,5 +1,6 @@ -use parquet2::encoding::Encoding; -use parquet2::{metadata::ColumnDescriptor, page::CompressedDataPage, write::WriteOptions}; +use parquet2::{ + encoding::Encoding, metadata::ColumnDescriptor, page::CompressedDataPage, write::WriteOptions, +}; use super::super::{levels, utils}; use super::basic::{build_statistics, encode_plain};