From 286d6c9e638cae83aafb9e15f88de34cd8e1cddc Mon Sep 17 00:00:00 2001 From: "Jorge C. Leitao" Date: Tue, 12 Apr 2022 04:42:25 +0000 Subject: [PATCH] Made offsets be always written --- src/write/file.rs | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/write/file.rs b/src/write/file.rs index cb54bf8db..e800fc71b 100644 --- a/src/write/file.rs +++ b/src/write/file.rs @@ -127,7 +127,7 @@ impl FileWriter { let num_rows = self.row_groups.iter().map(|group| group.num_rows).sum(); if self.options.write_statistics { - // write column indexes + // write column indexes (require page statistics) self.row_groups .iter_mut() .zip(self.page_specs.iter()) @@ -144,25 +144,27 @@ impl FileWriter { )?; Result::Ok(()) })?; - - // write offset index - self.row_groups - .iter_mut() - .zip(self.page_specs.iter()) - .try_for_each(|(group, pages)| { - group.columns.iter_mut().zip(pages.iter()).try_for_each( - |(column, pages)| { - let offset = self.offset; - column.offset_index_offset = Some(offset as i64); - self.offset += write_offset_index(&mut self.writer, pages)?; - column.offset_index_length = Some((self.offset - offset) as i32); - Result::Ok(()) - }, - )?; - Result::Ok(()) - })?; }; + // write offset index + self.row_groups + .iter_mut() + .zip(self.page_specs.iter()) + .try_for_each(|(group, pages)| { + group + .columns + .iter_mut() + .zip(pages.iter()) + .try_for_each(|(column, pages)| { + let offset = self.offset; + column.offset_index_offset = Some(offset as i64); + self.offset += write_offset_index(&mut self.writer, pages)?; + column.offset_index_length = Some((self.offset - offset) as i32); + Result::Ok(()) + })?; + Result::Ok(()) + })?; + let metadata = FileMetaData::new( self.options.version.into(), self.schema.into_thrift(),