From f893cb333b17b1c8362e5ca6653baadb71b043e2 Mon Sep 17 00:00:00 2001 From: Aymeric Rateau Date: Thu, 25 Apr 2024 23:14:48 +0200 Subject: [PATCH] Mdf3 complex reading bug fix (#16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add python 3.12 and remove 3.8 * tick patch version * updated readme * fixed doc generation warnings * added anyhow error handling in python interface * use ArrayFormater from Arrow crate to display content of array * removed filtering of empty data, can happen at initialisation * merged linear and rational conversion from conversion4 only and reused in conversion3 * forgot default value assignment in case of value to text conversion * clippy * remove generated parquet file during test * add debuggging view, more robust in case of non UTF8 characters --------- Co-authored-by: «ratal» <«ratal@ratal.org»> --- .../py-mac-aarch64-apple-release.yml | 2 +- .github/workflows/py-release-manylinux.yml | 2 +- .../workflows/py-release-windows-macos.yml | 2 +- .vscode/launch.json | 2 +- Cargo.lock | 136 +++++++------- Cargo.toml | 3 +- README.md | 6 +- src/data_holder/channel_data.rs | 95 +--------- src/mdfinfo/sym_buf_reader.rs | 2 +- src/mdfr.rs | 175 ++++++++++-------- src/mdfreader/conversions3.rs | 95 ++-------- src/mdfreader/conversions4.rs | 92 ++++----- src/mdfreader/data_read3.rs | 4 +- src/tests.rs | 10 +- 14 files changed, 259 insertions(+), 367 deletions(-) diff --git a/.github/workflows/py-mac-aarch64-apple-release.yml b/.github/workflows/py-mac-aarch64-apple-release.yml index 2dd0109..d173b3f 100644 --- a/.github/workflows/py-mac-aarch64-apple-release.yml +++ b/.github/workflows/py-mac-aarch64-apple-release.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: ["macos-latest"] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: [ "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 - name: Install latest Rust nightly diff --git a/.github/workflows/py-release-manylinux.yml b/.github/workflows/py-release-manylinux.yml index d1ee02b..c160d24 100644 --- a/.github/workflows/py-release-manylinux.yml +++ b/.github/workflows/py-release-manylinux.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.8', '3.9', '3.10' , "3.11"] + python-version: [ '3.9', '3.10' , "3.11", "3.12"] steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 diff --git a/.github/workflows/py-release-windows-macos.yml b/.github/workflows/py-release-windows-macos.yml index 0d523cb..4ef3fee 100644 --- a/.github/workflows/py-release-windows-macos.yml +++ b/.github/workflows/py-release-windows-macos.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: os: ["macos-latest", "windows-latest"] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 - name: Install latest Rust nightly diff --git a/.vscode/launch.json b/.vscode/launch.json index 1535c10..832063f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -53,7 +53,7 @@ } }, "args": [ - "/home/ratal/workspace/mdfreader/mdfreader/tests/MDF4/ASAM_COMMON_MDF_V4-1-0/Base_Standard/Examples/Simple/test.mf4" + "/home/ratal/workspace/mdfreader/mdfreader/tests/mdf3/TGT.dat" ], "cwd": "${workspaceFolder}" }, diff --git a/Cargo.lock b/Cargo.lock index ca3f450..f2d718e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -528,7 +528,7 @@ checksum = "4da9a32f3fed317401fa3c862968128267c3106685286e15d5aaa3d7389c2f60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -570,12 +570,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" +checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" dependencies = [ "jobserver", "libc", + "once_cell", ] [[package]] @@ -938,7 +939,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -1239,9 +1240,9 @@ checksum = "9028f49264629065d057f340a86acb84867925865f73bbf8d47b4d149a7e88b8" [[package]] name = "jobserver" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" dependencies = [ "libc", ] @@ -1410,7 +1411,7 @@ dependencies = [ [[package]] name = "mdfr" -version = "0.6.0" +version = "0.6.1" dependencies = [ "anyhow", "arrow", @@ -1425,6 +1426,7 @@ dependencies = [ "encoding_rs", "env_logger", "fasteval", + "glob", "half", "itertools 0.12.1", "libc", @@ -1830,9 +1832,9 @@ dependencies = [ [[package]] name = "polars" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c352aaa0399c0863eecd879f2cbe585c9026c5cafe432f029025e4bec3adf43" +checksum = "0ea21b858b16b9c0e17a12db2800d11aa5b4bd182be6b3022eb537bbfc1f2db5" dependencies = [ "getrandom", "polars-arrow", @@ -1850,9 +1852,9 @@ dependencies = [ [[package]] name = "polars-arrow" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f88d3cfc6b500f106f03a5f4d37f700deef38c5b58a2e843edb3ec31f5a0ec19" +checksum = "725b09f2b5ef31279b66e27bbab63c58d49d8f6696b66b1f46c7eaab95e80f75" dependencies = [ "ahash", "atoi", @@ -1895,9 +1897,9 @@ dependencies = [ [[package]] name = "polars-compute" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf264bfb632aaeba859fe19a87fa051d850e72542c36177ccea71fd9cae84079" +checksum = "a796945b14b14fbb79b91ef0406e6fddca2be636e889f81ea5d6ee7d36efb4fe" dependencies = [ "bytemuck", "either", @@ -1911,9 +1913,9 @@ dependencies = [ [[package]] name = "polars-core" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cb72917958e82f29d604429ab55851f561c7cd336f7744a7360f9e50b9ac88" +checksum = "465f70d3e96b6d0b1a43c358ba451286b8c8bd56696feff020d65702aa33e35c" dependencies = [ "ahash", "bitflags 2.5.0", @@ -1944,9 +1946,9 @@ dependencies = [ [[package]] name = "polars-error" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c18ef81979a6d9e9fdbd25ad3bf1591cbd5c474489f785af44604cf591cd636d" +checksum = "5224d5d05e6b8a6f78b75951ae1b5f82c8ab1979e11ffaf5fd41941e3d5b0757" dependencies = [ "polars-arrow-format", "regex", @@ -1956,9 +1958,9 @@ dependencies = [ [[package]] name = "polars-io" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d47f2cdd8e2a2bfc71b0d30444d4c378ddc0d6f80826746fc3c731c06251b42" +checksum = "b2c8589e418cbe4a48228d64b2a8a40284a82ec3c98817c0c2bcc0267701338b" dependencies = [ "ahash", "atoi_simd", @@ -1986,9 +1988,9 @@ dependencies = [ [[package]] name = "polars-lazy" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee5683b551f5e2bb004468edec0f87fd585f436c2e5f89b292c2bfee1b6f5d4f" +checksum = "89b2632b1af668e2058d5f8f916d8fbde3cac63d03ae29a705f598e41dcfeb7f" dependencies = [ "ahash", "bitflags 2.5.0", @@ -2009,9 +2011,9 @@ dependencies = [ [[package]] name = "polars-ops" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f311543e0e110d385867df25f47c1c740ee0cc854feead54262a24b0246383bb" +checksum = "efdbdb4d9a92109bc2e0ce8e17af5ae8ab643bb5b7ee9d1d74f0aeffd1fbc95f" dependencies = [ "ahash", "argminmax", @@ -2039,9 +2041,9 @@ dependencies = [ [[package]] name = "polars-parquet" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41cd1f445fea8377350dfa2bd216785839ce97c826299c7e0e9557c1dbe887f" +checksum = "b421d2196f786fdfe162db614c8485f8308fe41575d4de634a39bbe460d1eb6a" dependencies = [ "ahash", "base64 0.21.7", @@ -2058,9 +2060,9 @@ dependencies = [ [[package]] name = "polars-pipe" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f57de92c0ca9851e89cf9374cd88029f9bb2197937c34d571ec2a7ac45cca3" +checksum = "48700f1d5bd56a15451e581f465c09541492750360f18637b196f995470a015c" dependencies = [ "crossbeam-channel", "crossbeam-queue", @@ -2083,9 +2085,9 @@ dependencies = [ [[package]] name = "polars-plan" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c509bc273c402a8b1fbfa63df2b2e90ca10d30decab698c7739003817de67e1" +checksum = "2fb8e2302e20c44defd5be8cad9c96e75face63c3a5f609aced8c4ec3b3ac97d" dependencies = [ "ahash", "bytemuck", @@ -2109,9 +2111,9 @@ dependencies = [ [[package]] name = "polars-row" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "695a9954f5aa273e44c497c19f806177f787ccf87cd4b3044c96a5057266a861" +checksum = "a515bdc68c2ae3702e3de70d89601f3b71ca8137e282a226dddb53ee4bacfa2e" dependencies = [ "bytemuck", "polars-arrow", @@ -2121,9 +2123,9 @@ dependencies = [ [[package]] name = "polars-sql" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7cdf3b41bda70004ed3ec78652eb690aec3db5d99dfac03fbf9995fe76a7e26" +checksum = "7b4bb7cc1c04c3023d1953b2f1dec50515e8fd8169a5a2bf4967b3b082232db7" dependencies = [ "hex", "polars-arrow", @@ -2139,9 +2141,9 @@ dependencies = [ [[package]] name = "polars-time" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bdc956b63e99a5ad1dabd9d397ce9ce50f703e503a5039c972968683a953d0c" +checksum = "efc18e3ad92eec55db89d88f16c22d436559ba7030cf76f86f6ed7a754b673f1" dependencies = [ "atoi", "chrono", @@ -2159,9 +2161,9 @@ dependencies = [ [[package]] name = "polars-utils" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355b126757b4a87da5248ae6eb644e99b5583a11ffc2d42e13b2b856d43e84be" +checksum = "c760b6c698cfe2fbbbd93d6cfb408db14ececfe1d92445dae2229ce1b5b21ae8" dependencies = [ "ahash", "bytemuck", @@ -2192,9 +2194,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e" +checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" dependencies = [ "unicode-ident", ] @@ -2257,7 +2259,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -2270,7 +2272,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -2374,7 +2376,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -2444,9 +2446,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.32" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ "bitflags 2.5.0", "errno", @@ -2496,22 +2498,22 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.197" +version = "1.0.198" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.198" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -2637,7 +2639,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -2650,7 +2652,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -2666,9 +2668,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.59" +version = "2.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a" +checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" dependencies = [ "proc-macro2", "quote", @@ -2677,9 +2679,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.30.10" +version = "0.30.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d7c217777061d5a2d652aea771fb9ba98b6dade657204b08c4b9604d11555b" +checksum = "87341a165d73787554941cd5ef55ad728011566fe714e987d1b976c15dbc3a83" dependencies = [ "cfg-if", "core-foundation-sys", @@ -2740,7 +2742,7 @@ checksum = "c8f546451eaa38373f549093fe9fd05e7d2bade739e2ddf834b9968621d60107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -2751,22 +2753,22 @@ checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -2931,7 +2933,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", "wasm-bindgen-shared", ] @@ -2953,7 +2955,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3003,11 +3005,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ - "winapi", + "windows-sys", ] [[package]] @@ -3194,7 +3196,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9caba2f..589a61c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mdfr" -version = "0.6.0" +version = "0.6.1" description = "A package for reading and writing MDF files" authors = ["ratal "] edition = "2021" @@ -63,6 +63,7 @@ optional = true [dev-dependencies] criterion = "0.5" # for benchmark test-log = "*" +glob = "*" [build-dependencies] cbindgen = "0.26" # to generate C api headers diff --git a/README.md b/README.md index 75c5324..8916dc7 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,10 @@ ASAM mdf reader in rust Currently a personal project to learn rust from past experience with python (mdfreader). mdfr is currently able to be used from python interpreter (using pyO3) to read mdf 3.x and 4.x files. You can install it with 'pip install mdfr' command. +It can also be used in pure Rust environment as polars, numpy, pyo3 are optional features Using rayon crate on many parts of the code allows to have faster parsing in a safe and easy way compared to python. -To allow efficient data sharing with many other tools, mdfr stores the data using arrow2. Polars (pandas equivalent) use is therefore also straight forward. +To allow efficient data sharing with many other tools, mdfr stores the data using arrow. Polars (pandas equivalent) use is therefore also straight forward. A C/C++ api is also available allowing to get list of channels, units, description and data using the Arrow CDataInterface. + It can be used the following way with python interpreter: ```python @@ -38,5 +40,3 @@ obj.export_to_parquet('file_name', compression_option) # write to mdf4 file, compressed or not obj.write('file_name', conpression_flag) ``` - -A C/C++ api is also available allowing to get list of channels, units, description and data using the Arrow CDataInterface. diff --git a/src/data_holder/channel_data.rs b/src/data_holder/channel_data.rs index 27e0235..ccaf8ea 100644 --- a/src/data_holder/channel_data.rs +++ b/src/data_holder/channel_data.rs @@ -12,6 +12,7 @@ use arrow::datatypes::{ DataType, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type, UInt32Type, UInt64Type, UInt8Type, }; +use arrow::util::display::{ArrayFormatter, FormatOptions}; use itertools::Itertools; use std::fmt; @@ -1228,7 +1229,7 @@ impl ChannelData { ChannelData::ArrayDFloat64(a) => a.nulls().is_some(), } } - /// converts the ChannelData into a ArrayRef (alis of Arc) + /// converts the ChannelData into a ArrayRef pub fn as_ref(&self) -> Arc { match self { ChannelData::Int8(a) => Arc::new(a.finish_cloned()) as ArrayRef, @@ -1698,91 +1699,13 @@ pub fn try_from(value: &dyn Array) -> Result { impl fmt::Display for ChannelData { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - ChannelData::Int8(array) => { - writeln!(f, "{array:?}") - } - ChannelData::UInt8(array) => { - writeln!(f, "{array:?}") - } - ChannelData::Int16(array) => { - writeln!(f, "{array:?}") - } - ChannelData::UInt16(array) => { - writeln!(f, "{array:?}") - } - ChannelData::Int32(array) => { - writeln!(f, "{array:?}") - } - ChannelData::UInt32(array) => { - writeln!(f, "{array:?}") - } - ChannelData::Float32(array) => { - writeln!(f, "{array:?}") - } - ChannelData::Int64(array) => { - writeln!(f, "{array:?}") - } - ChannelData::UInt64(array) => { - writeln!(f, "{array:?}") - } - ChannelData::Float64(array) => { - writeln!(f, "{array:?}") - } - ChannelData::Complex32(array) => { - writeln!(f, "{array:?}") - } - ChannelData::Complex64(array) => { - writeln!(f, "{array:?}") - } - ChannelData::Utf8(array) => { - for text in array.values_slice().iter() { - writeln!(f, " {text:?} ")?; - } - writeln!(f, " ") - } - ChannelData::VariableSizeByteArray(array) => { - for text in array.values_slice().iter() { - writeln!(f, " {text:?} ")?; - } - writeln!(f, " ") - } - ChannelData::FixedSizeByteArray(array) => { - for text in array.finish_cloned().iter() { - writeln!(f, " {text:?} ")?; - } - writeln!(f, " ") - } - ChannelData::ArrayDInt8(array) => { - writeln!(f, "{:?}", array) - } - ChannelData::ArrayDUInt8(array) => { - writeln!(f, "{:?}", array) - } - ChannelData::ArrayDInt16(array) => { - writeln!(f, "{:?}", array) - } - ChannelData::ArrayDUInt16(array) => { - writeln!(f, "{:?}", array) - } - ChannelData::ArrayDInt32(array) => { - writeln!(f, "{:?}", array) - } - ChannelData::ArrayDUInt32(array) => { - writeln!(f, "{:?}", array) - } - ChannelData::ArrayDFloat32(array) => { - writeln!(f, "{:?}", array) - } - ChannelData::ArrayDInt64(array) => { - writeln!(f, "{:?}", array) - } - ChannelData::ArrayDUInt64(array) => { - writeln!(f, "{:?}", array) - } - ChannelData::ArrayDFloat64(array) => { - writeln!(f, "{:?}", array) - } + let format_option = FormatOptions::new(); + let data = self.as_ref(); + let displayer = + ArrayFormatter::try_new(&data, &format_option).map_err(|_| std::fmt::Error)?; + for i in 0..self.len() { + write!(f, " {}", displayer.value(i))?; } + Ok(()) } } diff --git a/src/mdfinfo/sym_buf_reader.rs b/src/mdfinfo/sym_buf_reader.rs index beeb89c..45df846 100644 --- a/src/mdfinfo/sym_buf_reader.rs +++ b/src/mdfinfo/sym_buf_reader.rs @@ -20,7 +20,7 @@ where R: Read, { /// Creates a new SymBufReader with a specified `buffer_size`. - /// This newly created object wraps another object which is [Read](std::io::Read). + /// This newly created object wraps another object which is Read (std::io::Read). pub fn new(reader: R) -> Self { let buffer = vec![0; DEFAULT_BUF_SIZE]; Self { diff --git a/src/mdfr.rs b/src/mdfr.rs index eeb3b72..d9c8a3f 100644 --- a/src/mdfr.rs +++ b/src/mdfr.rs @@ -7,6 +7,7 @@ use crate::data_holder::channel_data::ChannelData; use crate::mdfinfo::MdfInfo; use crate::mdfreader::MasterSignature; use crate::mdfreader::Mdf; +use anyhow::Context; use arrow::array::ArrayData; use arrow::pyarrow::PyArrowType; use arrow::util::display::{ArrayFormatter, FormatOptions}; @@ -47,34 +48,38 @@ impl Mdfr { mdf.get_version() } /// returns channel's data, numpy array or list, depending if data type is numeric or string|bytes - fn get_channel_data(&self, channel_name: String) -> Py { + fn get_channel_data(&self, channel_name: String) -> PyResult> { let Mdfr(mdf) = self; // default py_array value is python None - pyo3::Python::with_gil(|py| { + let data = pyo3::Python::with_gil(|py| -> PyResult> { let mut py_array: Py; let dt = mdf.get_channel_data(&channel_name); if let Some(data) = dt { py_array = data.clone().into_py(py); if let Some(m) = data.clone().validity() { let mask: Py = m.iter().collect::>().into_py(py); - let locals = [("numpy", py.import("numpy").expect("could not import numpy"))] - .into_py_dict(py); + let locals = [( + "numpy", + py.import("numpy").context("could not import numpy")?, + )] + .into_py_dict(py); locals .set_item("py_array", &py_array) - .expect("cannot set python data"); + .context("cannot set python data")?; locals .set_item("mask", mask) - .expect("cannot set python mask"); + .context("cannot set python mask")?; py_array = py .eval(r#"numpy.ma.array(py_array, mask=mask)"#, None, Some(locals)) - .expect("masked array creation failed") + .context("masked array creation failed")? .into_py(py); } } else { py_array = Python::None(py); } - py_array - }) + Ok(py_array) + })?; + Ok(data) } /// returns channel's numpy dtype fn get_channel_dtype(&self, channel_name: String) -> Py { @@ -125,9 +130,9 @@ impl Mdfr { } /// returns polar dataframe including channel #[cfg(feature = "polars")] - fn get_polars_dataframe(&self, channel_name: String) -> Py { + fn get_polars_dataframe(&self, channel_name: String) -> PyResult> { let Mdfr(mdf) = self; - pyo3::Python::with_gil(|py| { + Python::with_gil(|py| { let mut py_dataframe = Python::None(py); let channel_list = mdf.mdf_info.get_channel_names_cg_set(&channel_name); let series_dict = PyDict::new(py); @@ -137,17 +142,17 @@ impl Mdfr { .set_item( channel.clone(), rust_arrow_to_py_series(channel_data.as_ref(), channel) - .expect("Could not convert to python series"), + .context("Could not convert to python series")?, ) - .expect("could not store the serie in dict"); + .context("could not store the serie in dict")?; } } if !series_dict.is_empty() { let locals = PyDict::new(py); locals .set_item("series", series_dict) - .expect("cannot set python series_list"); - py.import("polars").expect("Could import polars"); + .context("cannot set python series_list")?; + py.import("polars").context("Could import polars")?; py.run( r#" import polars @@ -156,21 +161,21 @@ df=polars.DataFrame(series) None, Some(locals), ) - .expect("dataframe creation failed"); + .context("dataframe creation failed")?; if let Ok(Some(df)) = locals.get_item("df") { py_dataframe = df.into(); } } - py_dataframe + Ok(py_dataframe) }) } /// returns channel's unit string - fn get_channel_unit(&self, channel_name: String) -> PyResult> { + fn get_channel_unit(&self, channel_name: String) -> PyResult> { let Mdfr(mdf) = self; - pyo3::Python::with_gil(|py| { + pyo3::Python::with_gil(|_py| { let unit_or_error = mdf.mdf_info.get_channel_unit(&channel_name); match unit_or_error { - Ok(unit) => Ok(unit.to_object(py)), + Ok(unit) => Ok(unit), Err(_) => Err(PyUnicodeDecodeError::new_err( "Invalid UTF-8 sequence in metadata", )), @@ -178,12 +183,12 @@ df=polars.DataFrame(series) }) } /// returns channel's description string - fn get_channel_desc(&self, channel_name: String) -> PyResult> { + fn get_channel_desc(&self, channel_name: String) -> PyResult> { let Mdfr(mdf) = self; - pyo3::Python::with_gil(|py| { + pyo3::Python::with_gil(|_py| { let desc_or_error = mdf.mdf_info.get_channel_desc(&channel_name); match desc_or_error { - Ok(desc) => Ok(desc.to_object(py)), + Ok(desc) => Ok(desc), Err(_) => Err(PyUnicodeDecodeError::new_err( "Invalid UTF-8 sequence in metadata", )), @@ -199,7 +204,7 @@ df=polars.DataFrame(series) }) } /// returns channel's master data, numpy array or list, depending if data type is numeric or string|bytes - fn get_channel_master_data(&mut self, channel_name: String) -> Py { + fn get_channel_master_data(&mut self, channel_name: String) -> PyResult> { // default py_array value is python None let master = self.get_channel_master(channel_name); self.get_channel_data(master.to_string()) @@ -274,7 +279,7 @@ df=polars.DataFrame(series) let Mdfr(mdf) = self; pyo3::Python::with_gil(|_| -> Result<(), PyErr> { let array = array_to_rust(data) - .expect("data modification failed, could not extract numpy array"); + .context("data modification failed, could not extract numpy array")?; mdf.add_channel( channel_name, array, @@ -445,50 +450,54 @@ df=polars.DataFrame(series) }) } /// plot one channel - pub fn plot(&self, channel_name: String) { + pub fn plot(&self, channel_name: String) -> PyResult<()> { let Mdfr(mdf) = self; - pyo3::Python::with_gil(|py| { + pyo3::Python::with_gil(|py| -> PyResult<()> { let locals = PyDict::new(py); locals .set_item("channel_name", &channel_name) - .expect("cannot set python channel_name"); + .context("cannot set python channel_name")?; locals .set_item( "channel_unit", mdf.mdf_info.get_channel_unit(&channel_name).unwrap_or(None), ) - .expect("cannot set python channel_unit"); + .context("cannot set python channel_unit")?; if let Some(master_name) = mdf.mdf_info.get_channel_master(&channel_name) { locals .set_item("master_channel_name", &master_name) - .expect("cannot set python master_channel_name"); + .context("cannot set python master_channel_name")?; locals .set_item( "master_channel_unit", mdf.mdf_info.get_channel_unit(&master_name).unwrap_or(None), ) - .expect("cannot set python master_channel_unit"); - let data = self.get_channel_data(master_name); + .context("cannot set python master_channel_unit")?; + let data = self + .get_channel_data(master_name) + .context("failed getting master channel data")?; locals .set_item("master_data", data) - .expect("cannot set python master_data"); + .context("cannot set python master_data")?; } else { locals .set_item("master_channel_name", py.None()) - .expect("cannot set python master_channel_name"); + .context("cannot set python master_channel_name")?; locals .set_item("master_channel_unit", py.None()) - .expect("cannot set python master_channel_unit"); + .context("cannot set python master_channel_unit")?; locals .set_item("master_data", py.None()) - .expect("cannot set python master_data"); + .context("cannot set python master_data")?; } - let data = self.get_channel_data(channel_name); + let data = self + .get_channel_data(channel_name) + .context("failed getting channel data")?; locals .set_item("channel_data", data) - .expect("cannot set python channel_data"); + .context("cannot set python channel_data")?; py.import("matplotlib") - .expect("Could not import matplotlib"); + .context("Could not import matplotlib")?; py.run( r#" from matplotlib import pyplot @@ -508,15 +517,11 @@ pyplot.show() None, Some(locals), ) - .expect("plot python script failed"); + .context("plot python script failed")?; + Ok(()) }) } - // /// export to Parquet file - // pub fn export_to_parquet(&mut self, file_name: &str, compression_option: Option<&str>) { - // let Mdfr(mdf) = self; - // mdf.export_to_parquet(file_name, compression_option) - // .expect("could not export to parquet") - // } + /// display a representation of mdfinfo object content fn __repr__(&mut self) -> PyResult { let mut output: String; let format_option = FormatOptions::new(); @@ -528,50 +533,58 @@ pyplot.show() "Header :\n Author: {} Organisation:{}", mdfinfo3.hd_block.hd_author, mdfinfo3.hd_block.hd_organization ) - .expect("cannot print author and organisation"); + .context("cannot print author and organisation")?; writeln!( output, "Project: {} Subject:{}", mdfinfo3.hd_block.hd_project, mdfinfo3.hd_block.hd_subject ) - .expect("cannot print project and subject"); + .context("cannot print project and subject")?; writeln!( output, "Date: {:?} Time:{:?}", mdfinfo3.hd_block.hd_date, mdfinfo3.hd_block.hd_time ) - .expect("cannot print date and time"); - write!(output, "Comments: {}", mdfinfo3.hd_comment).expect("cannot print comments"); + .context("cannot print date and time")?; + write!(output, "Comments: {}", mdfinfo3.hd_comment) + .context("cannot print comments")?; for (master, list) in mdfinfo3.get_master_channel_names_set().iter() { if let Some(master_name) = master { writeln!(output, "\nMaster: {master_name}") - .expect("cannot print master channel name"); + .context("cannot print master channel name")?; } else { writeln!(output, "\nWithout Master channel") - .expect("cannot print thre is no master channel"); + .context("cannot print thre is no master channel")?; } for channel in list.iter() { - let unit = self.get_channel_unit(channel.to_string())?; - let desc = self.get_channel_desc(channel.to_string())?; - write!(output, " {channel} ").expect("cannot print channel name"); + let unit = self + .get_channel_unit(channel.to_string()) + .context("failed printing channel unit")? + .unwrap_or_default(); + let desc = self + .get_channel_desc(channel.to_string()) + .context("failed printing channel description")? + .unwrap_or_default(); + write!(output, " {channel} ").context("cannot print channel name")?; if let Some(data) = self.0.get_channel_data(channel) { if !data.is_empty() { let array = &data.as_ref(); let displayer = ArrayFormatter::try_new(array, &format_option) - .expect("failed creating formatter for arrow array"); + .context("failed creating formatter for arrow array")?; write!(&mut output, "{}", displayer.value(0)) - .expect("failed writing first value of array"); - write!(output, " ").expect("cannot print simple space character"); + .context("failed writing first value of array")?; + write!(output, " ") + .context("cannot print simple space character")?; write!(&mut output, "{}", displayer.value(data.len() - 1)) - .expect("failed writing last value of array"); + .context("failed writing last value of array")?; } writeln!( output, - " {unit} {desc} " - ).expect("cannot print channel unit and description with first and last item"); + " {:?} {:?} ",unit, desc + ).context("cannot print channel unit and description with first and last item")?; } else { - writeln!(output, " {unit} {desc} ") - .expect("cannot print channel unit and description"); + writeln!(output, " {:?} {:?} ", unit, desc) + .context("cannot print channel unit and description")?; } } } @@ -579,44 +592,50 @@ pyplot.show() } MdfInfo::V4(mdfinfo4) => { output = format!("Version : {}\n", mdfinfo4.id_block.id_ver); - writeln!(output, "{}", mdfinfo4.hd_block).expect("cannot print header block"); + writeln!(output, "{}", mdfinfo4.hd_block).context("cannot print header block")?; let comments = &mdfinfo4 .sharable .get_comments(mdfinfo4.hd_block.hd_md_comment); for c in comments.iter() { - writeln!(output, "{} {}", c.0, c.1).expect("cannot print header comments"); + writeln!(output, "{} {}", c.0, c.1).context("cannot print header comments")?; } for (master, list) in mdfinfo4.get_master_channel_names_set().iter() { if let Some(master_name) = master { writeln!(output, "\nMaster: {master_name}") - .expect("cannot print master channel name"); + .context("cannot print master channel name")?; } else { writeln!(output, "\nWithout Master channel") - .expect("cannot print thre is no master channel"); + .context("cannot print thre is no master channel")?; } for channel in list.iter() { - let unit = self.get_channel_unit(channel.to_string())?; - let desc = self.get_channel_desc(channel.to_string())?; - write!(output, " {channel} ").expect("cannot print channel name"); + let unit = self + .get_channel_unit(channel.to_string()) + .context("failed printing channel unit")? + .unwrap_or_default(); + let desc = self + .get_channel_desc(channel.to_string()) + .context("failed printing channel description")? + .unwrap_or_default(); + write!(output, " {channel} ").context("cannot print channel name")?; if let Some(data) = self.0.get_channel_data(channel) { if !data.is_empty() { let array = &data.as_ref(); let displayer = ArrayFormatter::try_new(array, &format_option) - .expect("failed creating formatter for arrow array"); + .context("failed creating formatter for arrow array")?; write!(&mut output, "{}", displayer.value(0)) - .expect("cannot print channel data"); + .context("cannot print channel data")?; write!(output, " .. ") - .expect("cannot print simple space character"); + .context("cannot print simple space character")?; write!(&mut output, "{}", displayer.value(data.len() - 1)) - .expect("cannot channel data"); + .context("cannot channel data")?; } writeln!( output, - " {unit} {desc} " - ).expect("cannot print channel unit and description with first and last item"); + " {:?} {:?} ",unit, desc + ).context("cannot print channel unit and description with first and last item")?; } else { - writeln!(output, " {unit} {desc} ") - .expect("cannot print channel unit and description"); + writeln!(output, " {:?} {:?} ", unit, desc) + .context("cannot print channel unit and description")?; } } } diff --git a/src/mdfreader/conversions3.rs b/src/mdfreader/conversions3.rs index dd7ce7b..1cbff3c 100644 --- a/src/mdfreader/conversions3.rs +++ b/src/mdfreader/conversions3.rs @@ -11,6 +11,7 @@ use std::collections::BTreeMap; use crate::data_holder::channel_data::ChannelData; use crate::mdfinfo::mdfinfo3::{Cn3, Conversion, Dg3, SharableBlocks3}; +use crate::mdfreader::conversions4::{linear_calculation, rational_calculation}; use fasteval::Evaler; use fasteval::{Compiler, Instruction, Slab}; use log::warn; @@ -69,9 +70,11 @@ pub fn convert_all_channels(dg: &mut Dg3, sharable: &SharableBlocks3) -> Result< })? } Conversion::TextTable(cc_val_ref) => { - value_to_text(cn, cc_val_ref, &cycle_count).with_context(|| { - format!("value to text conversion failed for {}", cn.unique_name) - })? + if !cc_val_ref.is_empty() { + value_to_text(cn, cc_val_ref, &cycle_count).with_context(|| { + format!("value to text conversion failed for {}", cn.unique_name) + })? + } } Conversion::TextRangeTable(cc_val_ref) => { value_range_to_text(cn, cc_val_ref, &cycle_count).with_context(|| { @@ -86,36 +89,6 @@ pub fn convert_all_channels(dg: &mut Dg3, sharable: &SharableBlocks3) -> Result< Ok(()) } -/// Generic function calculating exponential conversion -#[inline] -fn linear_calculation( - array: &mut PrimitiveBuilder, - cc_val: &[f64], -) -> Result, Error> -where - ::Native: AsPrimitive, - T::Native: NumCast, -{ - let p1 = cc_val[0]; - let p2 = cc_val[1]; - let array_f64: Float64Array = array - .finish() - .try_unary(|value| { - num::cast::cast::(value).ok_or_else(|| { - ArrowError::CastError(format!("Can't cast value {:?} to f64", value,)) - }) - }) - .context("failed converting array to f64")?; - let mut out = Float64Builder::with_capacity(array.capacity()); - out.values_slice_mut() - .iter_mut() - .zip(array_f64.values()) - .for_each(|(y, x)| { - *y = *x * p2 + p1; - }); - Ok(out) -} - /// Apply linear conversion to get physical data fn linear_conversion(cn: &mut Cn3, cc_val: &[f64]) -> Result<(), Error> { let p1 = cc_val[0]; @@ -124,61 +97,61 @@ fn linear_conversion(cn: &mut Cn3, cc_val: &[f64]) -> Result<(), Error> { match &mut cn.data { ChannelData::UInt8(a) => { cn.data = ChannelData::Float64( - linear_calculation(a, cc_val) + linear_calculation(a, p1, p2) .context("failed linear conversion of u8 channel")?, ); } ChannelData::Int8(a) => { cn.data = ChannelData::Float64( - linear_calculation(a, cc_val) + linear_calculation(a, p1, p2) .context("failed linear conversion of i8 channel")?, ); } ChannelData::Int16(a) => { cn.data = ChannelData::Float64( - linear_calculation(a, cc_val) + linear_calculation(a, p1, p2) .context("failed linear conversion of i16 channel")?, ); } ChannelData::UInt16(a) => { cn.data = ChannelData::Float64( - linear_calculation(a, cc_val) + linear_calculation(a, p1, p2) .context("failed linear conversion of u16 channel")?, ); } ChannelData::Int32(a) => { cn.data = ChannelData::Float64( - linear_calculation(a, cc_val) + linear_calculation(a, p1, p2) .context("failed linear conversion of i32 channel")?, ); } ChannelData::UInt32(a) => { cn.data = ChannelData::Float64( - linear_calculation(a, cc_val) + linear_calculation(a, p1, p2) .context("failed linear conversion of u32 channel")?, ); } ChannelData::Float32(a) => { cn.data = ChannelData::Float64( - linear_calculation(a, cc_val) + linear_calculation(a, p1, p2) .context("failed linear conversion of f32 channel")?, ); } ChannelData::Int64(a) => { cn.data = ChannelData::Float64( - linear_calculation(a, cc_val) + linear_calculation(a, p1, p2) .context("failed linear conversion of i64 channel")?, ); } ChannelData::UInt64(a) => { cn.data = ChannelData::Float64( - linear_calculation(a, cc_val) + linear_calculation(a, p1, p2) .context("failed linear conversion of u64 channel")?, ); } ChannelData::Float64(a) => { cn.data = ChannelData::Float64( - linear_calculation(a, cc_val) + linear_calculation(a, p1, p2) .context("failed linear conversion of f64 channel")?, ); } @@ -191,40 +164,6 @@ fn linear_conversion(cn: &mut Cn3, cc_val: &[f64]) -> Result<(), Error> { Ok(()) } -/// Generic function calculating rational conversion -#[inline] -fn rational_calculation( - array: &mut PrimitiveBuilder, - cc_val: &[f64], -) -> Result, Error> -where - ::Native: AsPrimitive, - T::Native: NumCast, -{ - let p1 = cc_val[0]; - let p2 = cc_val[1]; - let p3 = cc_val[2]; - let p4 = cc_val[3]; - let p5 = cc_val[4]; - let p6 = cc_val[5]; - let array_f64: Float64Array = array - .finish() - .try_unary(|value| { - num::cast::cast::(value).ok_or_else(|| { - ArrowError::CastError(format!("Can't cast value {:?} to f64", value,)) - }) - }) - .context("failed converting array to f64")?; - let mut out = Float64Builder::with_capacity(array.capacity()); - out.values_slice_mut() - .iter_mut() - .zip(array_f64.values()) - .for_each(|(y, x)| { - *y = (x * x * p1 + x * p2 + p3) / (x * x * p4 + x * p5 + p6); - }); - Ok(out) -} - /// Apply rational conversion to get physical data fn rational_conversion(cn: &mut Cn3, cc_val: &[f64]) -> Result<(), Error> { match &mut cn.data { @@ -1088,6 +1027,8 @@ where let matched_key = cc_val_ref.iter().find(|&x| x.0 == *val); if let Some(key) = matched_key { new_array.append_value(key.1.clone()); + } else { + new_array.append_value(cc_val_ref[0].1.clone()); } }); Ok(new_array) diff --git a/src/mdfreader/conversions4.rs b/src/mdfreader/conversions4.rs index f52d04a..e1a5350 100644 --- a/src/mdfreader/conversions4.rs +++ b/src/mdfreader/conversions4.rs @@ -166,7 +166,7 @@ pub fn convert_all_channels(dg: &mut Dg4, sharable: &SharableBlocks) -> Result<( /// Generic function calculating linear expression #[inline] -fn linear_conversion_primitive( +pub fn linear_calculation( array: &mut PrimitiveBuilder, p1: f64, p2: f64, @@ -199,81 +199,81 @@ fn linear_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { match &mut cn.data { ChannelData::UInt8(a) => { cn.data = ChannelData::Float64( - linear_conversion_primitive(a, p1, p2) + linear_calculation(a, p1, p2) .context("failed linear conversion of u8 channel")?, ); } ChannelData::Int8(a) => { cn.data = ChannelData::Float64( - linear_conversion_primitive(a, p1, p2) + linear_calculation(a, p1, p2) .context("failed linear conversion of i8 channel")?, ); } ChannelData::Int16(a) => { cn.data = ChannelData::Float64( - linear_conversion_primitive(a, p1, p2) + linear_calculation(a, p1, p2) .context("failed linear conversion of i16 channel")?, ); } ChannelData::UInt16(a) => { cn.data = ChannelData::Float64( - linear_conversion_primitive(a, p1, p2) + linear_calculation(a, p1, p2) .context("failed linear conversion of u16 channel")?, ); } ChannelData::Int32(a) => { cn.data = ChannelData::Float64( - linear_conversion_primitive(a, p1, p2) + linear_calculation(a, p1, p2) .context("failed linear conversion of i32 channel")?, ); } ChannelData::UInt32(a) => { cn.data = ChannelData::Float64( - linear_conversion_primitive(a, p1, p2) + linear_calculation(a, p1, p2) .context("failed linear conversion of u32 channel")?, ); } ChannelData::Float32(a) => { cn.data = ChannelData::Float64( - linear_conversion_primitive(a, p1, p2) + linear_calculation(a, p1, p2) .context("failed linear conversion of f32 channel")?, ); } ChannelData::Int64(a) => { cn.data = ChannelData::Float64( - linear_conversion_primitive(a, p1, p2) + linear_calculation(a, p1, p2) .context("failed linear conversion of u16 channel")?, ); } ChannelData::UInt64(a) => { cn.data = ChannelData::Float64( - linear_conversion_primitive(a, p1, p2) + linear_calculation(a, p1, p2) .context("failed linear conversion of u64 channel")?, ); } ChannelData::Float64(a) => { cn.data = ChannelData::Float64( - linear_conversion_primitive(a, p1, p2) + linear_calculation(a, p1, p2) .context("failed linear conversion of f64 channel")?, ); } ChannelData::Complex32(a) => { cn.data = ChannelData::Complex64(ComplexArrow::new_from_primitive( - linear_conversion_primitive(a.values(), p1, p2) + linear_calculation(a.values(), p1, p2) .context("failed linear conversion of complex f32 channel")?, a.nulls(), )); } ChannelData::Complex64(a) => { cn.data = ChannelData::Complex64(ComplexArrow::new_from_primitive( - linear_conversion_primitive(a.values(), p1, p2) + linear_calculation(a.values(), p1, p2) .context("failed linear conversion of complex f64 channel")?, a.nulls(), )); } ChannelData::ArrayDUInt8(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - linear_conversion_primitive(a.values(), p1, p2) + linear_calculation(a.values(), p1, p2) .context("failed linear conversion of tensor u8 channel")?, a.nulls(), a.shape().clone(), @@ -282,7 +282,7 @@ fn linear_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDInt8(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - linear_conversion_primitive(a.values(), p1, p2) + linear_calculation(a.values(), p1, p2) .context("failed linear conversion of tensor i8 channel")?, a.nulls(), a.shape().clone(), @@ -291,7 +291,7 @@ fn linear_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDInt16(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - linear_conversion_primitive(a.values(), p1, p2) + linear_calculation(a.values(), p1, p2) .context("failed linear conversion of tensor i16 channel")?, a.nulls(), a.shape().clone(), @@ -300,7 +300,7 @@ fn linear_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDUInt16(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - linear_conversion_primitive(a.values(), p1, p2) + linear_calculation(a.values(), p1, p2) .context("failed linear conversion of tensor u16 channel")?, a.nulls(), a.shape().clone(), @@ -309,7 +309,7 @@ fn linear_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDInt32(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - linear_conversion_primitive(a.values(), p1, p2) + linear_calculation(a.values(), p1, p2) .context("failed linear conversion of tensor i32 channel")?, a.nulls(), a.shape().clone(), @@ -318,7 +318,7 @@ fn linear_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDUInt32(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - linear_conversion_primitive(a.values(), p1, p2) + linear_calculation(a.values(), p1, p2) .context("failed linear conversion of tensor u16 channel")?, a.nulls(), a.shape().clone(), @@ -327,7 +327,7 @@ fn linear_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDFloat32(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - linear_conversion_primitive(a.values(), p1, p2) + linear_calculation(a.values(), p1, p2) .context("failed linear conversion of tensor f32 channel")?, a.nulls(), a.shape().clone(), @@ -336,7 +336,7 @@ fn linear_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDInt64(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - linear_conversion_primitive(a.values(), p1, p2) + linear_calculation(a.values(), p1, p2) .context("failed linear conversion of tensor i64 channel")?, a.nulls(), a.shape().clone(), @@ -345,7 +345,7 @@ fn linear_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDUInt64(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - linear_conversion_primitive(a.values(), p1, p2) + linear_calculation(a.values(), p1, p2) .context("failed linear conversion of tensor u64 channel")?, a.nulls(), a.shape().clone(), @@ -354,7 +354,7 @@ fn linear_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDFloat64(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - linear_conversion_primitive(a.values(), p1, p2) + linear_calculation(a.values(), p1, p2) .context("failed linear conversion of tensor f64 channel")?, a.nulls(), a.shape().clone(), @@ -372,7 +372,7 @@ fn linear_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { /// Generic function calculating rational expression for a primitive #[inline] -fn rational_conversion_primitive( +pub fn rational_calculation( array: &PrimitiveBuilder, cc_val: &[f64], ) -> Result, Error> @@ -403,81 +403,81 @@ fn rational_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { match &mut cn.data { ChannelData::UInt8(a) => { cn.data = ChannelData::Float64( - rational_conversion_primitive(a, cc_val) + rational_calculation(a, cc_val) .context("failed rational conversion of u8 channel")?, ); } ChannelData::Int8(a) => { cn.data = ChannelData::Float64( - rational_conversion_primitive(a, cc_val) + rational_calculation(a, cc_val) .context("failed rational conversion of i8 channel")?, ); } ChannelData::Int16(a) => { cn.data = ChannelData::Float64( - rational_conversion_primitive(a, cc_val) + rational_calculation(a, cc_val) .context("failed rational conversion of i16 channel")?, ); } ChannelData::UInt16(a) => { cn.data = ChannelData::Float64( - rational_conversion_primitive(a, cc_val) + rational_calculation(a, cc_val) .context("failed rational conversion of u16 channel")?, ); } ChannelData::Int32(a) => { cn.data = ChannelData::Float64( - rational_conversion_primitive(a, cc_val) + rational_calculation(a, cc_val) .context("failed rational conversion of i32 channel")?, ); } ChannelData::UInt32(a) => { cn.data = ChannelData::Float64( - rational_conversion_primitive(a, cc_val) + rational_calculation(a, cc_val) .context("failed rational conversion of u32 channel")?, ); } ChannelData::Float32(a) => { cn.data = ChannelData::Float64( - rational_conversion_primitive(a, cc_val) + rational_calculation(a, cc_val) .context("failed rational conversion of f32 channel")?, ); } ChannelData::Int64(a) => { cn.data = ChannelData::Float64( - rational_conversion_primitive(a, cc_val) + rational_calculation(a, cc_val) .context("failed rational conversion of i64 channel")?, ); } ChannelData::UInt64(a) => { cn.data = ChannelData::Float64( - rational_conversion_primitive(a, cc_val) + rational_calculation(a, cc_val) .context("failed rational conversion of u64 channel")?, ); } ChannelData::Float64(a) => { cn.data = ChannelData::Float64( - rational_conversion_primitive(a, cc_val) + rational_calculation(a, cc_val) .context("failed rational conversion of f64 channel")?, ); } ChannelData::Complex32(a) => { cn.data = ChannelData::Complex64(ComplexArrow::new_from_primitive( - rational_conversion_primitive(a.values(), cc_val) + rational_calculation(a.values(), cc_val) .context("failed rational conversion of complex f32 channel")?, a.nulls(), )); } ChannelData::Complex64(a) => { cn.data = ChannelData::Complex64(ComplexArrow::new_from_primitive( - rational_conversion_primitive(a.values(), cc_val) + rational_calculation(a.values(), cc_val) .context("failed rational conversion of complex f64 channel")?, a.nulls(), )); } ChannelData::ArrayDUInt8(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - rational_conversion_primitive(a.values(), cc_val) + rational_calculation(a.values(), cc_val) .context("failed rational conversion of u8 tensor channel")?, a.nulls(), a.shape().clone(), @@ -486,7 +486,7 @@ fn rational_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDInt8(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - rational_conversion_primitive(a.values(), cc_val) + rational_calculation(a.values(), cc_val) .context("failed rational conversion of i8 tensor channel")?, a.nulls(), a.shape().clone(), @@ -495,7 +495,7 @@ fn rational_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDInt16(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - rational_conversion_primitive(a.values(), cc_val) + rational_calculation(a.values(), cc_val) .context("failed rational conversion of i16 tensor channel")?, a.nulls(), a.shape().clone(), @@ -504,7 +504,7 @@ fn rational_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDUInt16(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - rational_conversion_primitive(a.values(), cc_val) + rational_calculation(a.values(), cc_val) .context("failed rational conversion of u16 tensor channel")?, a.nulls(), a.shape().clone(), @@ -513,7 +513,7 @@ fn rational_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDInt32(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - rational_conversion_primitive(a.values(), cc_val) + rational_calculation(a.values(), cc_val) .context("failed rational conversion of i32 tensor channel")?, a.nulls(), a.shape().clone(), @@ -522,7 +522,7 @@ fn rational_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDUInt32(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - rational_conversion_primitive(a.values(), cc_val) + rational_calculation(a.values(), cc_val) .context("failed rational conversion of u32 tensor channel")?, a.nulls(), a.shape().clone(), @@ -531,7 +531,7 @@ fn rational_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDFloat32(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - rational_conversion_primitive(a.values(), cc_val) + rational_calculation(a.values(), cc_val) .context("failed rational conversion of f32 tensor channel")?, a.nulls(), a.shape().clone(), @@ -540,7 +540,7 @@ fn rational_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDInt64(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - rational_conversion_primitive(a.values(), cc_val) + rational_calculation(a.values(), cc_val) .context("failed rational conversion of i64 tensor channel")?, a.nulls(), a.shape().clone(), @@ -549,7 +549,7 @@ fn rational_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDUInt64(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - rational_conversion_primitive(a.values(), cc_val) + rational_calculation(a.values(), cc_val) .context("failed rational conversion of u64 tensor channel")?, a.nulls(), a.shape().clone(), @@ -558,7 +558,7 @@ fn rational_conversion(cn: &mut Cn4, cc_val: &[f64]) -> Result<(), Error> { } ChannelData::ArrayDFloat64(a) => { cn.data = ChannelData::ArrayDFloat64(TensorArrow::new_from_primitive( - rational_conversion_primitive(a.values(), cc_val) + rational_calculation(a.values(), cc_val) .context("failed rational conversion of f64 tensor channel")?, a.nulls(), a.shape().clone(), diff --git a/src/mdfreader/data_read3.rs b/src/mdfreader/data_read3.rs index 3e75ed1..f054669 100644 --- a/src/mdfreader/data_read3.rs +++ b/src/mdfreader/data_read3.rs @@ -22,9 +22,7 @@ pub fn read_channels_from_bytes( // iterates for each channel in parallel with rayon crate channels .par_iter_mut() - .filter(|(_cn_record_position, cn)| { - channel_names_to_read_in_dg.contains(&cn.unique_name) && !cn.data.is_empty() - }) + .filter(|(_cn_record_position, cn)| channel_names_to_read_in_dg.contains(&cn.unique_name)) .try_for_each(|(_cn_pos, cn): (&u32, &mut Cn3)| -> Result<(), Error> { let mut value: &[u8]; // value of channel at record let pos_byte_beg = cn.pos_byte_beg as usize; diff --git a/src/tests.rs b/src/tests.rs index 7c2e78e..86585a9 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -10,6 +10,7 @@ mod tests { use crate::data_holder::channel_data::ChannelData; use crate::mdfreader::Mdf; + use glob::glob; use std::fs; use std::io; use std::path::Path; @@ -18,6 +19,7 @@ mod tests { static BASE_PATH_MDF4: &str = "/home/ratal/workspace/mdfreader/mdfreader/tests/MDF4/ASAM_COMMON_MDF_V4-1-0/Base_Standard/Examples/"; static BASE_PATH_MDF3: &str = "/home/ratal/workspace/mdfreader/mdfreader/tests/mdf3/"; + static BASE_TEST_PATH: &str = "/home/ratal/workspace/mdfr/test_files"; static WRITING_MDF_FILE: &str = "/home/ratal/workspace/mdfr/test_files/test.mf4"; static WRITING_PARQUET_FILE: &str = "/home/ratal/workspace/mdfr/test_files/test_parquet"; @@ -1172,6 +1174,7 @@ mod tests { "{}{}", BASE_PATH_MDF4, &"Simple/PCV_iO_Gen3_LK1__3l_TDI.mf4" ); + let extension = "*.parquet"; let mut mdf = Mdf::new(&file)?; mdf.load_all_channels_data_in_memory()?; mdf.export_to_parquet(&WRITING_PARQUET_FILE, Some("zstd")) @@ -1184,7 +1187,12 @@ mod tests { let mut mdf = Mdf::new(&file)?; mdf.load_all_channels_data_in_memory()?; mdf.export_to_parquet(&WRITING_PARQUET_FILE, Some("snappy")) - .expect("failed writing mdf4 parquet file"); + .expect("failed writing mdf3 parquet file"); + // remove all generated parquet files + let pattern = format!("{}/{}", BASE_TEST_PATH, extension); + for path in glob(&pattern).unwrap().filter_map(Result::ok) { + fs::remove_file(path)?; + } Ok(()) } }