From c2a970ae2a1e1c60b8b676a185a00109778ba1f6 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Wed, 22 Dec 2021 19:02:36 -0800 Subject: [PATCH] fix local object store --- datafusion/src/datasource/object_store/local.rs | 10 +++++++--- datafusion/src/datasource/object_store/mod.rs | 8 ++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/datafusion/src/datasource/object_store/local.rs b/datafusion/src/datasource/object_store/local.rs index 84c838f13988..49274cb4179d 100644 --- a/datafusion/src/datasource/object_store/local.rs +++ b/datafusion/src/datasource/object_store/local.rs @@ -33,6 +33,8 @@ use crate::error::Result; use super::{ObjectReaderStream, SizedFile}; +impl ReadSeek for std::fs::File {} + #[derive(Debug)] /// Local File System as Object Store. pub struct LocalFileSystem; @@ -78,18 +80,20 @@ impl ObjectReader for LocalFileReader { ) } + fn sync_reader(&self) -> Result> { + Ok(Box::new(File::open(&self.file.path)?)) + } + fn sync_chunk_reader( &self, start: u64, length: usize, - ) -> Result> { + ) -> Result> { // A new file descriptor is opened for each chunk reader. // This okay because chunks are usually fairly large. let mut file = File::open(&self.file.path)?; file.seek(SeekFrom::Start(start))?; - let file = BufReader::new(file.take(length as u64)); - Ok(Box::new(file)) } diff --git a/datafusion/src/datasource/object_store/mod.rs b/datafusion/src/datasource/object_store/mod.rs index ff9842da9add..42a11010d117 100644 --- a/datafusion/src/datasource/object_store/mod.rs +++ b/datafusion/src/datasource/object_store/mod.rs @@ -35,6 +35,8 @@ use crate::error::{DataFusionError, Result}; trait ReadSeek: Read + Seek {} +impl ReadSeek for std::io::BufReader {} + /// Object Reader for one file in an object store. /// /// Note that the dynamic dispatch on the reader might @@ -50,12 +52,10 @@ pub trait ObjectReader: Send + Sync { &self, start: u64, length: usize, - ) -> Result>; + ) -> Result>; /// Get reader for the entire file - fn sync_reader(&self) -> Result> { - self.sync_chunk_reader(0, self.length() as usize) - } + fn sync_reader(&self) -> Result>; /// Get the size of the file fn length(&self) -> u64;