Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Added support to read json to BinaryArray #304

Merged
merged 1 commit into from
Aug 20, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions src/io/json/read/deserialize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,7 @@ use serde_json::Value;

use crate::types::NaturalDataType;
use crate::{
array::{
Array, BooleanArray, DictionaryArray, DictionaryKey, ListArray, NullArray, Offset,
PrimitiveArray, StructArray, Utf8Array,
},
array::*,
bitmap::MutableBitmap,
buffer::MutableBuffer,
datatypes::{DataType, IntervalUnit},
Expand Down Expand Up @@ -90,6 +87,14 @@ fn read_primitive<T: NativeType + NaturalDataType + NumCast>(
PrimitiveArray::from_trusted_len_iter(iter).to(data_type)
}

fn read_binary<O: Offset>(rows: &[&Value]) -> BinaryArray<O> {
let iter = rows.iter().map(|row| match row {
Value::String(v) => Some(v.as_bytes()),
_ => None,
});
BinaryArray::from_trusted_len_iter(iter)
}

fn read_boolean(rows: &[&Value]) -> BooleanArray {
let iter = rows.iter().map(|row| match row {
Value::Bool(v) => Some(v),
Expand Down Expand Up @@ -236,9 +241,8 @@ pub fn read(rows: &[&Value], data_type: DataType) -> Arc<dyn Array> {
DataType::LargeUtf8 => Arc::new(read_utf8::<i64>(rows)),
DataType::List(_) => Arc::new(read_list::<i32>(rows, data_type)),
DataType::LargeList(_) => Arc::new(read_list::<i64>(rows, data_type)),
// unsupported
//DataType::Binary => Box::new(BinaryArray::<i32>::new_empty()),
//DataType::LargeBinary => Box::new(BinaryArray::<i64>::new_empty()),
DataType::Binary => Arc::new(read_binary::<i32>(rows)),
DataType::LargeBinary => Arc::new(read_binary::<i64>(rows)),
DataType::Struct(_) => Arc::new(read_struct(rows, data_type)),
DataType::Dictionary(key_type, _) => match key_type.as_ref() {
DataType::Int8 => Arc::new(read_dictionary::<i8>(rows, data_type)),
Expand Down
10 changes: 8 additions & 2 deletions tests/it/io/json/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -167,20 +167,26 @@ fn case_basics() -> (String, Schema, Vec<Box<dyn Array>>) {
}

fn case_basics_schema() -> (String, Schema, Vec<Box<dyn Array>>) {
let data = r#"{"a":1, "b":2.0, "c":false, "d":"4"}
{"a":10, "b":-3.5, "c":true, "d":null}
let data = r#"{"a":1, "b":2.0, "c":false, "d":"4", "e":"4"}
{"a":10, "b":-3.5, "c":true, "d":null, "e":"text"}
{"a":100000000, "b":0.6, "d":"text"}"#
.to_string();
let schema = Schema::new(vec![
Field::new("a", DataType::UInt32, true),
Field::new("b", DataType::Float32, true),
Field::new("c", DataType::Boolean, true),
// note how "d" is not here
Field::new("e", DataType::Binary, true),
]);
let columns = vec![
Box::new(UInt32Array::from_slice(&[1, 10, 100000000])) as Box<dyn Array>,
Box::new(Float32Array::from_slice(&[2.0, -3.5, 0.6])),
Box::new(BooleanArray::from(&[Some(false), Some(true), None])),
Box::new(BinaryArray::<i32>::from(&[
Some(b"4".as_ref()),
Some(b"text".as_ref()),
None,
])),
];
(data, schema, columns)
}
Expand Down