Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
Fixed JSON order
Browse files Browse the repository at this point in the history
  • Loading branch information
jorgecarleitao committed Aug 5, 2022
1 parent 497d431 commit ffa59c8
Show file tree
Hide file tree
Showing 4 changed files with 22 additions and 3 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ regex-syntax = { version = "^0.6", optional = true }
streaming-iterator = { version = "0.1", optional = true }
fallible-streaming-iterator = { version = "0.1", optional = true }

json-deserializer = { version = "0.3", optional = true }
json-deserializer = { version = "0.4", optional = true, features = ["preserve_order"] }
indexmap = { version = "^1.6", optional = true }

# used to print columns in a nice columnar format
Expand Down
3 changes: 1 addition & 2 deletions src/io/json/read/infer_schema.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
use std::borrow::Borrow;
use std::collections::BTreeMap;

use indexmap::map::IndexMap as HashMap;
use indexmap::set::IndexSet as HashSet;
Expand Down Expand Up @@ -30,7 +29,7 @@ fn filter_map_nulls(dt: DataType) -> Option<DataType> {
}
}

fn infer_object(inner: &BTreeMap<String, Value>) -> Result<DataType> {
fn infer_object(inner: &HashMap<String, Value>) -> Result<DataType> {
let fields = inner
.iter()
.filter_map(|(key, value)| {
Expand Down
1 change: 1 addition & 0 deletions src/io/ndjson/read/file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ pub fn infer<R: std::io::BufRead>(
let mut data_types = HashSet::new();
while let Some(rows) = reader.next()? {
let value = parse(rows[0].as_bytes())?; // 0 because it is row by row
println!("{value:?}");
let data_type = infer_json(&value)?;
if data_type != DataType::Null {
data_types.insert(data_type);
Expand Down
19 changes: 19 additions & 0 deletions tests/it/io/ndjson/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -311,3 +311,22 @@ fn case(case: &str) -> (String, Box<dyn Array>) {
_ => todo!(),
}
}

#[test]
fn infer_object() -> Result<()> {
let data = r#"{"i64": 1, "f64": 0.1, "utf8": "foo1", "bools": true}
{"i64": 2, "f64": 0.2, "utf8": "foo2", "bools": false}
{"i64": 3, "f64": 0.3, "utf8": "foo3"}
{"i64": 4, "f64": 0.4, "utf8": "foo4", "bools": false}
"#;
let u64_fld = Field::new("i64", DataType::Int64, true);
let f64_fld = Field::new("f64", DataType::Float64, true);
let utf8_fld = Field::new("utf8", DataType::Utf8, true);
let bools_fld = Field::new("bools", DataType::Boolean, true);

let expected = DataType::Struct(vec![u64_fld, f64_fld, utf8_fld, bools_fld]);
let actual = infer(data)?;

assert_eq!(expected, actual);
Ok(())
}

0 comments on commit ffa59c8

Please sign in to comment.