Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
Updated examples.
Browse files Browse the repository at this point in the history
  • Loading branch information
jorgecarleitao committed Nov 13, 2021
1 parent 480e023 commit 2a48bbf
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 7 deletions.
6 changes: 3 additions & 3 deletions examples/parquet_read.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,13 @@ fn read_field(path: &str, row_group: usize, field: usize) -> Result<Box<dyn Arra
// yields an iterator of compressed pages. There is almost no CPU work in iterating.
let columns = read::get_column_iterator(&mut file, &metadata, row_group, field, None, vec![]);

// get the columns' logical type
let data_type = arrow_schema.fields()[field].data_type().clone();
// get the columns' field
let field = &arrow_schema.fields()[field];

// This is the actual work. In this case, pages are read and
// decompressed, decoded and deserialized to arrow.
// Because `columns` is an iterator, it uses a combination of IO and CPU.
let (array, _, _) = read::column_iter_to_array(columns, data_type, vec![])?;
let (array, _, _) = read::column_iter_to_array(columns, field, vec![])?;

Ok(array)
}
Expand Down
8 changes: 4 additions & 4 deletions examples/parquet_read_parallel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,14 +64,14 @@ fn parallel_read(path: &str, row_group: usize) -> Result<RecordBatch> {
let arrow_schema_consumer = arrow_schema.clone();
thread::spawn(move || {
let mut arrays = vec![];
while let Ok((field_i, field, column_chunks)) = rx_consumer.recv() {
while let Ok((field_i, parquet_field, column_chunks)) = rx_consumer.recv() {
let start = SystemTime::now();
let data_type = arrow_schema_consumer.fields()[field_i].data_type().clone();
let field = &arrow_schema_consumer.fields()[field_i];
println!("consumer {} start - {}", i, field_i);

let columns = read::ReadColumnIterator::new(field, column_chunks);
let columns = read::ReadColumnIterator::new(parquet_field, column_chunks);

let array = read::column_iter_to_array(columns, data_type, vec![]).map(|x| x.0);
let array = read::column_iter_to_array(columns, field, vec![]).map(|x| x.0);
println!(
"consumer {} end - {:?}: {}",
i,
Expand Down

0 comments on commit 2a48bbf

Please sign in to comment.