Skip to content

Commit

Permalink
fix local object store
Browse files Browse the repository at this point in the history
  • Loading branch information
houqp committed Dec 23, 2021
1 parent 0837507 commit c2a970a
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 7 deletions.
10 changes: 7 additions & 3 deletions datafusion/src/datasource/object_store/local.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ use crate::error::Result;

use super::{ObjectReaderStream, SizedFile};

impl ReadSeek for std::fs::File {}

#[derive(Debug)]
/// Local File System as Object Store.
pub struct LocalFileSystem;
Expand Down Expand Up @@ -78,18 +80,20 @@ impl ObjectReader for LocalFileReader {
)
}

fn sync_reader(&self) -> Result<Box<dyn ReadSeek + Send + Sync>> {
Ok(Box::new(File::open(&self.file.path)?))
}

fn sync_chunk_reader(
&self,
start: u64,
length: usize,
) -> Result<Box<dyn ReadSeek + Send + Sync>> {
) -> Result<Box<dyn Read + Send + Sync>> {
// A new file descriptor is opened for each chunk reader.
// This okay because chunks are usually fairly large.
let mut file = File::open(&self.file.path)?;
file.seek(SeekFrom::Start(start))?;

let file = BufReader::new(file.take(length as u64));

Ok(Box::new(file))
}

Expand Down
8 changes: 4 additions & 4 deletions datafusion/src/datasource/object_store/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ use crate::error::{DataFusionError, Result};

trait ReadSeek: Read + Seek {}

impl<R: Read + Seek> ReadSeek for std::io::BufReader<R> {}

/// Object Reader for one file in an object store.
///
/// Note that the dynamic dispatch on the reader might
Expand All @@ -50,12 +52,10 @@ pub trait ObjectReader: Send + Sync {
&self,
start: u64,
length: usize,
) -> Result<Box<dyn ReadSeek + Send + Sync>>;
) -> Result<Box<dyn Read + Send + Sync>>;

/// Get reader for the entire file
fn sync_reader(&self) -> Result<Box<dyn ReadSeek + Send + Sync>> {
self.sync_chunk_reader(0, self.length() as usize)
}
fn sync_reader(&self) -> Result<Box<dyn ReadSeek + Send + Sync>>;

/// Get the size of the file
fn length(&self) -> u64;
Expand Down

0 comments on commit c2a970a

Please sign in to comment.