Skip to content

Commit

Permalink
Cherry pick of Support binary data type in build_struct_array (#705)
Browse files Browse the repository at this point in the history
* Support binary data type in `build_struct_array`. (#702)

* Support binary data type in `build_struct_array`.

* Modify test case.

* cargo fmt

Co-authored-by: Andrew Lamb <[email protected]>

* Remove accidentally included test

Co-authored-by: Yuan Zhou <[email protected]>
  • Loading branch information
alamb and zijie0 authored Aug 23, 2021
1 parent 412a5bd commit 4ca0d95
Showing 1 changed file with 40 additions and 0 deletions.
40 changes: 40 additions & 0 deletions arrow/src/json/reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1224,6 +1224,14 @@ impl Decoder {
})
.collect::<StringArray>(),
) as ArrayRef),
DataType::Binary => Ok(Arc::new(
rows.iter()
.map(|row| {
let maybe_value = row.get(field.name());
maybe_value.and_then(|value| value.as_str())
})
.collect::<BinaryArray>(),
) as ArrayRef),
DataType::List(ref list_field) => {
match list_field.data_type() {
DataType::Dictionary(ref key_ty, _) => {
Expand Down Expand Up @@ -2963,6 +2971,38 @@ mod tests {
assert_eq!(batch.num_rows(), 3);
}

#[test]
fn test_json_read_binary_structs() {
let schema = Schema::new(vec![Field::new("c1", DataType::Binary, true)]);
let decoder = Decoder::new(Arc::new(schema), 1024, None);
let batch = decoder
.next_batch(
&mut vec![
Ok(serde_json::json!({
"c1": "₁₂₃",
})),
Ok(serde_json::json!({
"c1": "foo",
})),
]
.into_iter(),
)
.unwrap()
.unwrap();
let data = batch.columns().iter().collect::<Vec<_>>();

let schema = Schema::new(vec![Field::new("c1", DataType::Binary, true)]);
let binary_values = BinaryArray::from(vec!["₁₂₃".as_bytes(), "foo".as_bytes()]);
let expected_batch =
RecordBatch::try_new(Arc::new(schema), vec![Arc::new(binary_values)])
.unwrap();
let expected_data = expected_batch.columns().iter().collect::<Vec<_>>();

assert_eq!(data, expected_data);
assert_eq!(batch.num_columns(), 1);
assert_eq!(batch.num_rows(), 2);
}

#[test]
fn test_json_iterator() {
let builder = ReaderBuilder::new().infer_schema(None).with_batch_size(5);
Expand Down

0 comments on commit 4ca0d95

Please sign in to comment.