Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
MOD error on empty reader
Browse files Browse the repository at this point in the history
  • Loading branch information
Roberto-XY committed May 1, 2022
1 parent 426b0dc commit 53a313d
Show file tree
Hide file tree
Showing 3 changed files with 21 additions and 7 deletions.
7 changes: 7 additions & 0 deletions src/io/ndjson/read/deserialize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@ use super::super::super::json::read::_deserialize;
/// # Errors
/// This function errors iff any of the rows is not a valid JSON (i.e. the format is not valid NDJSON).
pub fn deserialize(rows: &[String], data_type: DataType) -> Result<Arc<dyn Array>, ArrowError> {
if rows.is_empty() {
return Err(ArrowError::ExternalFormat(
"Cannot deserialize 0 NDJSON rows because empty string is not a valid JSON value"
.to_string(),
));
}

// deserialize strings to `Value`s
let rows = rows
.iter()
Expand Down
6 changes: 6 additions & 0 deletions src/io/ndjson/read/file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,12 @@ pub fn infer<R: std::io::BufRead>(
reader: &mut R,
number_of_rows: Option<usize>,
) -> Result<DataType> {
if !reader.fill_buf().map(|b| !b.is_empty())? {
return Err(ArrowError::ExternalFormat(
"Cannot infer NDJSON types on empty reader because empty string is not a valid JSON value".to_string(),
));
}

let rows = vec!["".to_string(); 1]; // 1 <=> read row by row
let mut reader = FileReader::new(reader, rows, number_of_rows);

Expand Down
15 changes: 8 additions & 7 deletions tests/it/io/ndjson/read.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use std::sync::Arc;

use arrow2::array::*;
use arrow2::datatypes::{DataType, Field};
use arrow2::error::Result;
use arrow2::error::{ArrowError, Result};
use arrow2::io::ndjson::read as ndjson_read;
use arrow2::io::ndjson::read::FallibleStreamingIterator;

Expand Down Expand Up @@ -74,14 +74,15 @@ fn read_null() -> Result<()> {
#[test]
fn read_empty_reader() -> Result<()> {
let ndjson = "";
let expected_data_type = DataType::Null;

let data_type = infer(ndjson)?;
assert_eq!(expected_data_type, data_type);
let infer_error = infer(ndjson);
assert!(matches!(infer_error, Err(ArrowError::ExternalFormat(_))));

let arrays = read_and_deserialize(ndjson, &data_type, 1000)?;
let expected: Vec<Arc<dyn Array>> = vec![];
assert_eq!(expected, arrays);
let deserialize_error = ndjson_read::deserialize(&[], DataType::Null);
assert!(matches!(
deserialize_error,
Err(ArrowError::ExternalFormat(_))
));
Ok(())
}

Expand Down

0 comments on commit 53a313d

Please sign in to comment.