diff --git a/README.md b/README.md index 0e7e3834..b035b013 100644 --- a/README.md +++ b/README.md @@ -442,7 +442,7 @@ usage: zcbor code [-h] -c CDDL [--no-prelude] [-v] -t ENTRY_TYPES [ENTRY_TYPES ...] [-d] [-e] [--time-header] [--git-sha-header] [-b {32,64}] [--include-prefix INCLUDE_PREFIX] [-s] - [--file-header FILE_HEADER] + [--file-header FILE_HEADER] [--unordered-maps] Parse a CDDL file and produce C code that validates and xcodes CBOR. The output from this script is a C file and a header file. The header file @@ -543,6 +543,13 @@ options: generated files, e.g. copyright. Can be a string or a path to a file. If interpreted as a path to an existing file, the file's contents will be used. + --unordered-maps Add support in the generated code for parsing maps + with unknown element order. When enabled, the + generated code will use the zcbor_unordered_map_*() + API to decode data whenever inside a map. zcbor + detects when ZCBOR_MAP_SMART_SEARCH is needed and + enable This places restrictions on the level of + ambiguity allowed between map keys in a map. ``` diff --git a/samples/pet/src/pet_decode.c b/samples/pet/src/pet_decode.c index 4800f2c2..efee5776 100644 --- a/samples/pet/src/pet_decode.c +++ b/samples/pet/src/pet_decode.c @@ -20,6 +20,7 @@ #error "The type file was generated with a different default_max_qty than this file" #endif + #define log_result(state, result, func) do { \ if (!result) { \ zcbor_trace_file(state); \ @@ -62,7 +63,7 @@ int cbor_decode_Pet( struct Pet *result, size_t *payload_len_out) { - zcbor_state_t states[4]; + zcbor_state_t states[4 + 0]; struct zcbor_state_init_params params = { .states = states, @@ -70,6 +71,7 @@ int cbor_decode_Pet( .payload = payload, .payload_len = payload_len, .elem_count = 1, + .flags_bytes = 0, }; int ret = zcbor_entry_func((zcbor_decoder_t *)decode_Pet, (void *)result, ¶ms); diff --git a/samples/pet/src/pet_encode.c b/samples/pet/src/pet_encode.c index dac72da7..5b1784af 100644 --- a/samples/pet/src/pet_encode.c +++ b/samples/pet/src/pet_encode.c @@ -20,6 +20,7 @@ #error "The type file was generated with a different default_max_qty than this file" #endif + #define log_result(state, result, func) do { \ if (!result) { \ zcbor_trace_file(state); \ @@ -56,7 +57,7 @@ int cbor_encode_Pet( const struct Pet *input, size_t *payload_len_out) { - zcbor_state_t states[4]; + zcbor_state_t states[4 + 0]; struct zcbor_state_init_params params = { .states = states, @@ -64,6 +65,7 @@ int cbor_encode_Pet( .payload = payload, .payload_len = payload_len, .elem_count = 1, + .flags_bytes = 0, }; int ret = zcbor_entry_func((zcbor_decoder_t *)encode_Pet, (void *)input, ¶ms); diff --git a/src/zcbor_decode.c b/src/zcbor_decode.c index 85407968..b2a1b1c6 100644 --- a/src/zcbor_decode.c +++ b/src/zcbor_decode.c @@ -959,6 +959,10 @@ bool zcbor_unordered_map_search(zcbor_decoder_t key_decoder, zcbor_state_t *stat (void)old_flags; } + if (!should_try_key(state)) { + zcbor_log("Skipping element at index %zu.\n", get_current_index(state, 0)); + } + if (should_try_key(state) && try_key(state, key_result, key_decoder)) { if (!ZCBOR_MANUALLY_PROCESS_ELEM(state)) { ZCBOR_FAIL_IF(!zcbor_elem_processed(state)); @@ -1576,8 +1580,8 @@ bool zcbor_multi_decode(size_t min_decode, *num_decode = i; state->payload = payload_bak; state->elem_count = elem_count_bak; - ZCBOR_ERR_IF(i < min_decode, ZCBOR_ERR_ITERATIONS); zcbor_log("Found %zu elements.\r\n", i); + ZCBOR_ERR_IF(i < min_decode, ZCBOR_ERR_ITERATIONS); return true; } } diff --git a/tests/cases/serial_recovery.cddl b/tests/cases/serial_recovery.cddl index a8211a77..d25b6b47 100644 --- a/tests/cases/serial_recovery.cddl +++ b/tests/cases/serial_recovery.cddl @@ -4,13 +4,11 @@ ; SPDX-License-Identifier: Apache-2.0 ; -Member = ("image" => int) / - ("data" => bstr) / - ("len" => int) / - ("off" => int) / - ("sha" => bstr) / - (tstr => any) - Upload = { - 3*8members: Member + ? "image" => uint, + ? "len" => uint, + "off" => uint, + ? "sha" => bstr, + ? "data" => bstr, + ? "upgrade" => bool, } diff --git a/tests/cases/unordered_map.cddl b/tests/cases/unordered_map.cddl new file mode 100644 index 00000000..f53db45f --- /dev/null +++ b/tests/cases/unordered_map.cddl @@ -0,0 +1,46 @@ +; +; Copyright (c) 2023 Nordic Semiconductor ASA +; +; SPDX-License-Identifier: Apache-2.0 +; + +UnorderedMap1 = { + 1 => "one", + 2 => "two", + int => bstr, + "foo" => "bar", + bazboz, + *bstr => intlist, +} + +bazboz = ("baz" => "boz") +intlist = [*int] + +UnorderedMap2 = { + +typeUnion, + "map" => { + ?-1 => bstr, + ?-2 => bstr, + ?bool => int, + } +} + +typeUnion = type1 / type2 / typeDefault +type1 = 1 => tstr +type2 = 2 => tstr +typeDefault = int => nil + +UnorderedMap3 = { + 1*2000 uint => uint, + 1*2000 nint => nint, +} + +group1 = ( + uint => tstr, + nint => bstr, +) +type3 = 3*3group1 + +UnorderedMap4 = { + 3*3group1, +} diff --git a/tests/decode/test3_simple/CMakeLists.txt b/tests/decode/test3_simple/CMakeLists.txt index 6621eef1..dc19f5ef 100644 --- a/tests/decode/test3_simple/CMakeLists.txt +++ b/tests/decode/test3_simple/CMakeLists.txt @@ -35,6 +35,7 @@ set(py_command_serial_recovery -d ${bit_arg} --short-names + --unordered-maps # Testing the --include-prefix option --include-prefix serial diff --git a/tests/decode/test3_simple/src/main.c b/tests/decode/test3_simple/src/main.c index 0dd120ac..60ddd8db 100644 --- a/tests/decode/test3_simple/src/main.c +++ b/tests/decode/test3_simple/src/main.c @@ -487,19 +487,15 @@ ZTEST(cbor_decode_test3, test_serial1) zassert_equal(ZCBOR_SUCCESS, ret, "decoding failed: %d.", ret); zassert_equal(sizeof(serial_rec_input1), decode_len, NULL); - zassert_equal(5, upload.members_count, - "expect 5 members"); - zassert_equal(Member_data_c, upload.members[0].members - .Member_choice, "expect data 1st"); - zassert_equal(Member_image_c, upload.members[1].members - .Member_choice, "expect image 2nd"); - zassert_equal(Member_len_c, upload.members[2].members - .Member_choice, "was %d\r\n", upload.members[2].members - .Member_choice); - zassert_equal(Member_off_c, upload.members[3].members - .Member_choice, "expect off 4th"); - zassert_equal(Member_sha_c, upload.members[4].members - .Member_choice, "expect sha 5th"); + zassert_true(upload.data_present, NULL); + zassert_equal(0x129, upload.data.data.len, NULL); + zassert_true(upload.image_present, NULL); + zassert_equal(0, upload.image.image, NULL); + zassert_true(upload.len_present, NULL); + zassert_equal(0x3b2c, upload.len.len, NULL); + zassert_equal(0, upload.off, NULL); + zassert_true(upload.sha_present, NULL); + zassert_equal(0x20, upload.sha.sha.len, NULL); } ZTEST(cbor_decode_test3, test_serial2) @@ -511,18 +507,15 @@ ZTEST(cbor_decode_test3, test_serial2) zassert_equal(ZCBOR_SUCCESS, ret, "decoding failed: %d.", ret); zassert_equal(sizeof(serial_rec_input2), decode_len, NULL); - zassert_equal(5, upload.members_count, - "expect 5 members"); - zassert_equal(Member_data_c, upload.members[0].members - .Member_choice, "expect data 1st"); - zassert_equal(Member_image_c, upload.members[1].members - .Member_choice, "expect image 2nd"); - zassert_equal(Member_len_c, upload.members[2].members - .Member_choice, "expect len 3rd"); - zassert_equal(Member_off_c, upload.members[3].members - .Member_choice, "expect off 4th"); - zassert_equal(Member_sha_c, upload.members[4].members - .Member_choice, "expect sha 5th"); + zassert_true(upload.data_present, NULL); + zassert_equal(0x129, upload.data.data.len, NULL); + zassert_true(upload.image_present, NULL); + zassert_equal(0, upload.image.image, NULL); + zassert_true(upload.len_present, NULL); + zassert_equal(0x2fe0, upload.len.len, NULL); + zassert_equal(0, upload.off, NULL); + zassert_true(upload.sha_present, NULL); + zassert_equal(0x20, upload.sha.sha.len, NULL); } ZTEST_SUITE(cbor_decode_test3, NULL, NULL, NULL, NULL, NULL); diff --git a/tests/decode/test9_manifest14/CMakeLists.txt b/tests/decode/test9_manifest14/CMakeLists.txt index c6c4cbdd..ec596d82 100644 --- a/tests/decode/test9_manifest14/CMakeLists.txt +++ b/tests/decode/test9_manifest14/CMakeLists.txt @@ -8,6 +8,7 @@ cmake_minimum_required(VERSION 3.13.1) find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) project(test9_manifest14) +set(MAP_SEARCH_FLAGS ON) # Because of --unordered maps include(../../cmake/test_template.cmake) if (NOT MANIFEST) @@ -30,6 +31,7 @@ set(py_command SUIT_Common_Sequence -d ${bit_arg} + --unordered-maps ) execute_process( diff --git a/tests/decode/testA_unordered_map/CMakeLists.txt b/tests/decode/testA_unordered_map/CMakeLists.txt new file mode 100644 index 00000000..80160325 --- /dev/null +++ b/tests/decode/testA_unordered_map/CMakeLists.txt @@ -0,0 +1,40 @@ +# +# Copyright (c) 2021 Nordic Semiconductor ASA +# +# SPDX-License-Identifier: Apache-2.0 +# + +cmake_minimum_required(VERSION 3.13.1) + +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(test9_manifest14) +set(MAP_SEARCH_FLAGS ON) # Because of --unordered maps +set(MAP_SMART_SEARCH ON) +include(../../cmake/test_template.cmake) + +set(py_command + ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_LIST_DIR}/../../../zcbor/zcbor.py + code + -c ${CMAKE_CURRENT_LIST_DIR}/../../cases/unordered_map.cddl + --output-cmake ${PROJECT_BINARY_DIR}/unordered_map.cmake + -t + UnorderedMap1 + UnorderedMap2 + UnorderedMap3 + UnorderedMap4 + -d + ${bit_arg} + --unordered-maps + ) + +execute_process( + COMMAND + ${py_command} + COMMAND_ERROR_IS_FATAL ANY +) + +include(${PROJECT_BINARY_DIR}/unordered_map.cmake) + +target_link_libraries(unordered_map PRIVATE zephyr_interface) +target_link_libraries(app PRIVATE unordered_map) diff --git a/tests/decode/testA_unordered_map/prj.conf b/tests/decode/testA_unordered_map/prj.conf new file mode 100644 index 00000000..7cc649c9 --- /dev/null +++ b/tests/decode/testA_unordered_map/prj.conf @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Nordic Semiconductor ASA +# +# SPDX-License-Identifier: Apache-2.0 +# + +CONFIG_ZTEST=y +CONFIG_ZTEST_STACK_SIZE=131072 diff --git a/tests/decode/testA_unordered_map/src/main.c b/tests/decode/testA_unordered_map/src/main.c new file mode 100644 index 00000000..8e56d7b3 --- /dev/null +++ b/tests/decode/testA_unordered_map/src/main.c @@ -0,0 +1,259 @@ +/* + * Copyright (c) 2023 Nordic Semiconductor ASA + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include "unordered_map_decode.h" +#include + + + +ZTEST(cbor_decode_testA, test_unordered_map1) +{ + const uint8_t payload_unordered_map1_1[] = { + MAP(7), + 3, 0x40, + 0x41, 0, LIST(3), 1, 2, 3, END + 2, 0x63, 't', 'w', 'o', + 0x63, 'b', 'a', 'z', 0x63, 'b', 'o', 'z', + 0x41, 2, LIST(0), END + 1, 0x63, 'o', 'n', 'e', + 0x63, 'f', 'o', 'o', 0x63, 'b', 'a', 'r', + END + }; + + const uint8_t payload_unordered_map1_2[] = { + MAP(5), + 2, 0x63, 't', 'w', 'o', + 3, 0x40, + 1, 0x63, 'o', 'n', 'e', + 0x63, 'b', 'a', 'z', 0x63, 'b', 'o', 'z', + 0x63, 'f', 'o', 'o', 0x63, 'b', 'a', 'r', + END + }; + + const uint8_t payload_unordered_map1_inv3[] = { + MAP(5), + 2, 0x63, 't', 'w', 'o', + 3, 0x40, + 1, 0x63, 'o', 'n', 'f' /* here */, + 0x63, 'b', 'a', 'z', 0x63, 'b', 'o', 'z', + 0x63, 'f', 'o', 'o', 0x63, 'b', 'a', 'r', + END + }; + + const uint8_t payload_unordered_map1_inv4[] = { + MAP(6), + 3, 0x40, + 0x63, 'b', 'a', 'z', 0x63, 'b', 'o', 'z', + 0x41, 0, LIST(3), 1, 2, 0x40 /* here */, END + 0x63, 'f', 'o', 'o', 0x63, 'b', 'a', 'r', + 1, 0x63, 'o', 'n', 'e', + 2, 0x63, 't', 'w', 'o', + END + }; + + struct UnorderedMap1 unordered_map1; + + int err = cbor_decode_UnorderedMap1(payload_unordered_map1_1, sizeof(payload_unordered_map1_1), &unordered_map1, NULL); + zassert_equal(ZCBOR_SUCCESS, err, "%s %d\n", zcbor_error_str(err), err); + zassert_equal(3, unordered_map1.UnorderedMap1_intbstr_key); + zassert_equal(0, unordered_map1.UnorderedMap1_intbstr.len); + zassert_equal(2, unordered_map1.UnorderedMap1_intlist_m_count); + zassert_equal(1, unordered_map1.UnorderedMap1_intlist_m[0].UnorderedMap1_intlist_m_key.len); + zassert_equal(2, unordered_map1.UnorderedMap1_intlist_m[0].UnorderedMap1_intlist_m_key.value[0]); + zassert_equal(0, unordered_map1.UnorderedMap1_intlist_m[0].UnorderedMap1_intlist_m.intlist_int_count); + zassert_equal(1, unordered_map1.UnorderedMap1_intlist_m[1].UnorderedMap1_intlist_m_key.len); + zassert_equal(0, unordered_map1.UnorderedMap1_intlist_m[1].UnorderedMap1_intlist_m_key.value[0]); + zassert_equal(3, unordered_map1.UnorderedMap1_intlist_m[1].UnorderedMap1_intlist_m.intlist_int_count); + zassert_equal(1, unordered_map1.UnorderedMap1_intlist_m[1].UnorderedMap1_intlist_m.intlist_int[0]); + zassert_equal(2, unordered_map1.UnorderedMap1_intlist_m[1].UnorderedMap1_intlist_m.intlist_int[1]); + zassert_equal(3, unordered_map1.UnorderedMap1_intlist_m[1].UnorderedMap1_intlist_m.intlist_int[2]); + + err = cbor_decode_UnorderedMap1(payload_unordered_map1_2, sizeof(payload_unordered_map1_2), &unordered_map1, NULL); + zassert_equal(ZCBOR_SUCCESS, err, "%s %d\n", zcbor_error_str(err), err); + zassert_equal(3, unordered_map1.UnorderedMap1_intbstr_key); + zassert_equal(0, unordered_map1.UnorderedMap1_intbstr.len); + zassert_equal(0, unordered_map1.UnorderedMap1_intlist_m_count); + + err = cbor_decode_UnorderedMap1(payload_unordered_map1_inv3, sizeof(payload_unordered_map1_inv3), &unordered_map1, NULL); + zassert_equal(ZCBOR_ERR_WRONG_VALUE, err, "%s\n", zcbor_error_str(err)); + + err = cbor_decode_UnorderedMap1(payload_unordered_map1_inv4, sizeof(payload_unordered_map1_inv4), &unordered_map1, NULL); + zassert_equal(ZCBOR_ERR_ELEMS_NOT_PROCESSED, err, "%s\n", zcbor_error_str(err)); +} + + +ZTEST(cbor_decode_testA, test_unordered_map2) +{ + const uint8_t payload_unordered_map2_1[] = { + MAP(4), + 1, 0x61, 'a', + 0x63, 'm', 'a', 'p', MAP(3), + 0x20, 0x40, + 0x21, 0x40, + 0xF4, 0x18, 100, + END + 0x21, 0xF6, + 2, 0x61, 'b', + END + }; + + const uint8_t payload_unordered_map2_2[] = { + MAP(2), + 0x63, 'm', 'a', 'p', MAP(1), + 0xF5, 0x39, 0x12, 0x34, + END + 0x22, 0xF6, + END + }; + + const uint8_t payload_unordered_map2_3[] = { + MAP(3), + 2, 0x66, 'f', 'o', 'o', 'b', 'a', 'r', + 0x63, 'm', 'a', 'p', MAP(2), + 0x21, 0x40, + 0x20, 0x40, + END + 1, 0x60, + END + }; + + const uint8_t payload_unordered_map2_inv4[] = { + MAP(4), + 1, 0x61, 'a', + 0x63, 'm', 'a', 'p', MAP(3), + 0x20, 0x40, + 0x21, 0x40, + 0xF4, 0xF6 /* here */, + END + 0x21, 0xF6, + 2, 0x61, 'b', + END + }; + + const uint8_t payload_unordered_map2_inv5[] = { + MAP(4), + 1, 0x61, 'a', + 0x62, 'm', 'a', /* here */ MAP(3), + 0x20, 0x40, + 0x21, 0x40, + 0xF4, 0x18, 100, + END + 0x21, 0xF6, + 2, 0x61, 'b', + END + }; + + struct UnorderedMap2 unordered_map2; + + int err = cbor_decode_UnorderedMap2(payload_unordered_map2_1, sizeof(payload_unordered_map2_1), &unordered_map2, NULL); + zassert_equal(ZCBOR_SUCCESS, err, "%s %d\n", zcbor_error_str(err), err); + zassert_equal(3, unordered_map2.UnorderedMap2_typeUnion_m_count); + zassert_equal(typeUnion_type1_m_c, unordered_map2.UnorderedMap2_typeUnion_m[0].typeUnion_choice); + zassert_equal(typeUnion_type2_m_c, unordered_map2.UnorderedMap2_typeUnion_m[1].typeUnion_choice); + zassert_equal(typeUnion_typeDefault_m_c, unordered_map2.UnorderedMap2_typeUnion_m[2].typeUnion_choice); + zassert_equal(1, unordered_map2.UnorderedMap2_typeUnion_m[0].typeUnion_type1_m.type1.len); + zassert_equal('a', unordered_map2.UnorderedMap2_typeUnion_m[0].typeUnion_type1_m.type1.value[0]); + zassert_equal(1, unordered_map2.UnorderedMap2_typeUnion_m[1].typeUnion_type2_m.type2.len); + zassert_equal('b', unordered_map2.UnorderedMap2_typeUnion_m[1].typeUnion_type2_m.type2.value[0]); + zassert_equal(-2, unordered_map2.UnorderedMap2_typeUnion_m[2].typeUnion_typeDefault_m.typeDefault_key); + zassert_true(unordered_map2.map_nint1bstr_present); + zassert_true(unordered_map2.map_nint2bstr_present); + zassert_true(unordered_map2.map_boolint_present); + zassert_equal(0, unordered_map2.map_nint1bstr.map_nint1bstr.len); + zassert_equal(0, unordered_map2.map_nint2bstr.map_nint2bstr.len); + zassert_false(unordered_map2.map_boolint.UnorderedMap2_map_boolint_key); + zassert_equal(100, unordered_map2.map_boolint.map_boolint); + + err = cbor_decode_UnorderedMap2(payload_unordered_map2_2, sizeof(payload_unordered_map2_2), &unordered_map2, NULL); + zassert_equal(ZCBOR_SUCCESS, err, "%s %d\n", zcbor_error_str(err), err); + zassert_equal(1, unordered_map2.UnorderedMap2_typeUnion_m_count); + zassert_equal(typeUnion_typeDefault_m_c, unordered_map2.UnorderedMap2_typeUnion_m[0].typeUnion_choice); + zassert_equal(-3, unordered_map2.UnorderedMap2_typeUnion_m[0].typeUnion_typeDefault_m.typeDefault_key); + zassert_true(unordered_map2.map_boolint_present); + zassert_true(unordered_map2.map_boolint.UnorderedMap2_map_boolint_key); + zassert_equal(-0x1235, unordered_map2.map_boolint.map_boolint); + + err = cbor_decode_UnorderedMap2(payload_unordered_map2_3, sizeof(payload_unordered_map2_3), &unordered_map2, NULL); + zassert_equal(ZCBOR_SUCCESS, err, "%s %d\n", zcbor_error_str(err), err); + zassert_equal(2, unordered_map2.UnorderedMap2_typeUnion_m_count); + zassert_equal(typeUnion_type1_m_c, unordered_map2.UnorderedMap2_typeUnion_m[0].typeUnion_choice); + zassert_equal(typeUnion_type2_m_c, unordered_map2.UnorderedMap2_typeUnion_m[1].typeUnion_choice); + zassert_equal(0, unordered_map2.UnorderedMap2_typeUnion_m[0].typeUnion_type1_m.type1.len); + zassert_equal(6, unordered_map2.UnorderedMap2_typeUnion_m[1].typeUnion_type2_m.type2.len); + zassert_mem_equal("foobar", unordered_map2.UnorderedMap2_typeUnion_m[1].typeUnion_type2_m.type2.value, 6); + zassert_true(unordered_map2.map_nint1bstr_present); + zassert_true(unordered_map2.map_nint2bstr_present); + zassert_equal(0, unordered_map2.map_nint1bstr.map_nint1bstr.len); + zassert_equal(0, unordered_map2.map_nint2bstr.map_nint2bstr.len); + + err = cbor_decode_UnorderedMap2(payload_unordered_map2_inv4, sizeof(payload_unordered_map2_inv4), &unordered_map2, NULL); + zassert_equal(ZCBOR_ERR_ELEMS_NOT_PROCESSED, err, "%s %d\n", zcbor_error_str(err), err); + + err = cbor_decode_UnorderedMap2(payload_unordered_map2_inv5, sizeof(payload_unordered_map2_inv5), &unordered_map2, NULL); + zassert_equal(ZCBOR_ERR_ELEM_NOT_FOUND, err, "%s %d\n", zcbor_error_str(err), err); +} + + +ZTEST(cbor_decode_testA, test_unordered_map3) +{ + uint8_t payload_unordered_map3_1[21000]; + + ZCBOR_STATE_E(state_e, 1, payload_unordered_map3_1, sizeof(payload_unordered_map3_1), 0); + + zassert_true(zcbor_map_start_encode(state_e, 2000)); + for (uint32_t i = 1; i <= 1000; i++) { + zassert_true(zcbor_uint32_put(state_e, i), "%d\n", i); + zassert_true(zcbor_uint32_put(state_e, i), "%d\n", i); + zassert_true(zcbor_int32_put(state_e, -i), "%d\n", -i); + zassert_true(zcbor_int32_put(state_e, -i), "%d\n", -i); + } + zassert_true(zcbor_map_end_encode(state_e, 2000)); + + struct UnorderedMap3 unordered_map3; + + int err = cbor_decode_UnorderedMap3(payload_unordered_map3_1, sizeof(payload_unordered_map3_1), &unordered_map3, NULL); + zassert_equal(ZCBOR_SUCCESS, err, "%s %d\n", zcbor_error_str(err), err); + + zassert_equal(1000, unordered_map3.UnorderedMap3_uintuint_count, "%d != %d", 1000, unordered_map3.UnorderedMap3_uintuint_count); + zassert_equal(1000, unordered_map3.UnorderedMap3_nintnint_count, "%d != %d", 1000, unordered_map3.UnorderedMap3_nintnint_count); +} + + +ZTEST(cbor_decode_testA, test_unordered_map4) +{ + uint8_t payload_unordered_map4_1[50]; + + ZCBOR_STATE_E(state_e, 1, payload_unordered_map4_1, sizeof(payload_unordered_map4_1), 0); + + zassert_true(zcbor_map_start_encode(state_e, 6)); + for (int32_t i = -1; i >= -3; i--) { + zassert_true(zcbor_int32_put(state_e, i), "%d\n", i); + zassert_true(zcbor_bstr_put_lit(state_e, "world"), NULL); + } + for (uint32_t i = 1; i <= 3; i++) { + zassert_true(zcbor_uint32_put(state_e, i), "%d\n", i); + zassert_true(zcbor_tstr_put_lit(state_e, "hello"), NULL); + } + zassert_true(zcbor_map_end_encode(state_e, 6)); + + struct UnorderedMap4 unordered_map4; + + int err = cbor_decode_UnorderedMap4(payload_unordered_map4_1, sizeof(payload_unordered_map4_1), &unordered_map4, NULL); + zassert_equal(ZCBOR_SUCCESS, err, "%s %d\n", zcbor_error_str(err), err); + + zassert_equal(3, unordered_map4.UnorderedMap4_group1_m_count, "%d != %d", 3, unordered_map4.UnorderedMap4_group1_m_count); + for (uint32_t i = 1; i <= 3; i++) { + zassert_equal(i, unordered_map4.UnorderedMap4_group1_m[i - 1].UnorderedMap4_group1_m.group1_uinttstr_key); + zassert_equal(-i, unordered_map4.UnorderedMap4_group1_m[i - 1].UnorderedMap4_group1_m.group1_nintbstr_key); + } +} + + +ZTEST_SUITE(cbor_decode_testA, NULL, NULL, NULL, NULL, NULL); diff --git a/tests/decode/testA_unordered_map/testcase.yaml b/tests/decode/testA_unordered_map/testcase.yaml new file mode 100644 index 00000000..bf977536 --- /dev/null +++ b/tests/decode/testA_unordered_map/testcase.yaml @@ -0,0 +1,4 @@ +tests: + zcbor.decode.testA_unordered_map: + platform_allow: native_posix native_posix_64 mps2_an521 qemu_malta_be + tags: zcbor decode unordered_map testA diff --git a/zcbor/zcbor.py b/zcbor/zcbor.py index 93856aff..7cfb2020 100755 --- a/zcbor/zcbor.py +++ b/zcbor/zcbor.py @@ -1093,6 +1093,7 @@ def __init__(self, *args, **kwargs): # Used as a guard against endless recursion in self.dependsOn() self.dependsOnCall = False self.skipped = False + self.unordered_maps = False def var_name(self, with_prefix=False, observe_skipped=True): """Name of variables and enum members for this element.""" @@ -1323,7 +1324,9 @@ def single_func_impl_condition(self): or (self.tags and self in self.my_types.values()) or self.type_def_condition() or (self.type in ["LIST", "MAP"]) - or (self.type == "GROUP" and len(self.value) != 0)) + or (self.type == "GROUP" and len(self.value) != 0) + or (self.unordered_maps and self.is_key + and (self.repeated_single_func_impl_condition() or self.range_check_condition()))) def repeated_single_func_impl_condition(self): """Whether this element needs its own encoder/decoder function.""" @@ -1913,11 +1916,13 @@ class CddlTypes(NamedTuple): class CodeGenerator(CddlXcoder): """Class for generating C code that encode/decodes CBOR and validates it according to the CDDL. """ - def __init__(self, mode, entry_type_names, default_bit_size, *args, **kwargs): + def __init__(self, mode, entry_type_names, default_bit_size, *args, + unordered_maps=False, **kwargs): super(CodeGenerator, self).__init__(*args, **kwargs) self.mode = mode self.entry_type_names = entry_type_names self.default_bit_size = default_bit_size + self.unordered_maps = unordered_maps @classmethod def from_cddl(cddl_class, mode, *args, **kwargs): @@ -1942,6 +1947,11 @@ def is_cbor(self): def init_args(self): return (self.mode, self.entry_type_names, self.default_bit_size, self.default_max_qty) + def init_kwargs(self): + kwargs = {"unordered_maps": self.unordered_maps} + kwargs.update(super().init_kwargs()) + return kwargs + def delegate_type_condition(self): """Whether to use the C type of the first child as this type's C type""" ret = self.skip_condition() and (self.multi_var_condition() @@ -2373,6 +2383,28 @@ def num_backups(self): total += 1 return total + def num_map_search_flags(self): + """Calculate the number of map search flags needed for this element and all descendants.""" + total_this = 0 # Number of flags needed for this element directly + total_descendants = 0 # Number of flags needed for descendants + if self.key: + total_descendants += sum(self.key.num_map_search_flags()) + total_this += 1 + if self.cbor_var_condition(): + total_descendants += sum(self.cbor.num_map_search_flags()) + if self.type in ["LIST", "MAP", "GROUP", "UNION"]: + total_children = \ + max([sum(child.num_map_search_flags()) for child in self.value]) + if self.type == "MAP": + # Round up to the nearest byte + total_children = (total_children + 7) // 8 * 8 + total_descendants += total_children + if self.type == "OTHER": + t, t_ex = self.my_types[self.value].num_map_search_flags() + total_this += t + total_descendants += t_ex + return total_this * self.max_qty, total_descendants + def depends_on(self): """Return a number indicating how many other elements this element depends on. @@ -2424,17 +2456,63 @@ def list_counts(self): }[self.type]()) return retval + def is_multiple_elem_group(self): + """Recursively determine whether the current element is a GROUP with multiple elements.""" + if (self.type == "GROUP") and (len(self.value) > 1) and (self.max_qty != self.min_qty): + return True + if self.type == "OTHER": + if self.my_types[self.value].list_counts() != (1, 1) and (self.max_qty != self.min_qty): + return True + return self.my_types[self.value].is_multiple_elem_group() + return False + + def is_valid_map_elem(self): + """(Redefinition) Return whether the current element is a valid member of a MAP.""" + if self.unordered_maps: + if self.is_multiple_elem_group(): + return False, \ + "Groups with both variable repetitions and multiple elements are not allowed " \ + "in unordered maps." + return super().is_valid_map_elem() + + def elem_needs_map_smart_search(self, is_in_map): + """Recursively determine whether the current element needs ZCBOR_MAP_SMART_SEARCH defined""" + if is_in_map \ + and self.unordered_maps \ + and ((self.max_qty > 1) or (self.key and not self.key.is_unambiguous_repeated())): + return True + + if self.type == "OTHER" \ + and (self.value not in self.entry_type_names or is_in_map) \ + and self.my_types[self.value].elem_needs_map_smart_search(is_in_map): + return True + + # Validation of child elements. + if self.type in ["MAP", "LIST", "UNION", "GROUP"]: + for child in self.value: + if child.elem_needs_map_smart_search( + self.type != "LIST" and (is_in_map or self.type == "MAP")): + return True + if self.cbor: + if self.cbor.elem_needs_map_smart_search(False): + return True + def xcode_list(self): """Return the full code needed to encode/decode a "LIST" or "MAP" element with children.""" start_func = f"zcbor_{self.type.lower()}_start_{self.mode}" end_func = f"zcbor_{self.type.lower()}_end_{self.mode}" end_func_force = f"zcbor_list_map_end_force_{self.mode}" + if self.type == "MAP" and self.mode == "decode" and self.unordered_maps: + start_func = "zcbor_unordered_map_start_decode" + end_func = "zcbor_unordered_map_end_decode" assert start_func in [ "zcbor_list_start_decode", "zcbor_list_start_encode", - "zcbor_map_start_decode", "zcbor_map_start_encode"] + "zcbor_map_start_decode", "zcbor_map_start_encode", + "zcbor_unordered_map_start_decode"] assert end_func in [ "zcbor_list_end_decode", "zcbor_list_end_encode", - "zcbor_map_end_decode", "zcbor_map_end_encode"] + "zcbor_map_end_decode", "zcbor_map_end_encode", + "zcbor_unordered_map_end_decode"] assert self.type in ["LIST", "MAP"], \ "Expected LIST or MAP type, was %s." % self.type _, max_counts = zip( @@ -2487,11 +2565,16 @@ def xcode_union(self): for i in range(1, len(child_values)): if ((not self.value[i].is_int_disambiguated()) and self.value[i - 1].simple_func_condition()): - child_values[i] = f"(zcbor_union_elem_code(state) && {child_values[i]})" + if not self.value[i].key_var_condition() or not self.unordered_maps: + child_values[i] = f"(zcbor_union_elem_code(state) && {child_values[i]})" child_code = f"{newl_ind}|| ".join(child_values) - return f"(zcbor_union_start_code(state) "\ - + f"&& (int_res = ({child_code}), zcbor_union_end_code(state), int_res))" + if len(self.value) > 0 \ + and (not self.value[0].key_var_condition() or not self.unordered_maps): + return f"(zcbor_union_start_code(state) "\ + + f"&& (int_res = ({child_code}), zcbor_union_end_code(state), int_res))" + else: + return f"({child_code})" else: return ternary_if_chain( self.choice_var_access(), @@ -2611,12 +2694,20 @@ def do_xcode_single_func_prim(inner_union_int=None): }[self.type] xcoders = [] if self.key: - xcoders.append(self.key.full_xcode(union_int)) + if self.mode == "decode" and self.unordered_maps: + func, *arguments = self.key.single_func(self.key.val_access(), ptr_result=True) + x_args = xcode_args(*arguments) + xcoders.append( + f"zcbor_unordered_map_search((zcbor_{self.mode}r_t *){func}, {x_args})") + else: + xcoders.append(self.key.full_xcode(union_int=union_int)) if self.tags: xcoders.extend(self.xcode_tags()) if self.mode == "decode": xcoders.append(xcoder()) xcoders.extend(range_checks) + if self.key and self.unordered_maps: + xcoders.append("zcbor_elem_processed(state)") elif self.type == "BSTR" and self.cbor: xcoders.append(xcoder()) xcoders.extend(self.range_checks("tmp_str")) @@ -2728,6 +2819,8 @@ def __init__(self, entry_types, modes, print_time, default_max_qty, git_sha='', self.functions = dict() self.type_defs = dict() + self.needs_map_smart_search = {"encode": False, "decode": False} + # Sort type definitions so the typedefs will come in the correct order in the header file # and the function in the correct order in the c file. for mode in modes: @@ -2738,6 +2831,9 @@ def __init__(self, entry_types, modes, print_time, default_max_qty, git_sha='', self.functions[mode] = self.used_funcs(mode) self.type_defs[mode] = self.unique_types(mode) + self.needs_map_smart_search[mode] \ + = any(t.elem_needs_map_smart_search(False) for t in self.sorted_types[mode]) + self.version = __version__ if git_sha: @@ -2869,10 +2965,15 @@ def render_function(self, xcoder, mode): def render_entry_function(self, xcoder, mode): """Render a single entry function (API function) with signature and body.""" func_name, func_arg = (xcoder.xcode_func_name(), struct_ptr_name(mode)) + num_flag_states = "0" + num_flags = 0 + if xcoder.unordered_maps and mode == "decode": + num_flags = sum(xcoder.num_map_search_flags()) + num_flag_states = f"ZCBOR_FLAG_STATES({num_flags})" return f""" {xcoder.public_xcode_func_sig()} {{ - zcbor_state_t states[{xcoder.num_backups() + 2}]; + zcbor_state_t states[{xcoder.num_backups() + 2} + {num_flag_states}]; struct zcbor_state_init_params params = {{ .states = states, @@ -2880,6 +2981,7 @@ def render_entry_function(self, xcoder, mode): .payload = payload, .payload_len = payload_len, .elem_count = {xcoder.list_counts()[1]}, + .flags_bytes = {(num_flags + 7) // 8}, }}; int ret = zcbor_entry_func((zcbor_decoder_t *){func_name}, (void *){func_arg}, ¶ms); @@ -2895,6 +2997,13 @@ def render_file_header(self, line_prefix): lp = line_prefix return (f"\n{lp} " + self.file_header.replace("\n", f"\n{lp} ")).replace(" \n", "\n") + def render_smart_search_check(self): + return """ +#ifndef ZCBOR_MAP_SMART_SEARCH +#error "This file needs ZCBOR_MAP_SMART_SEARCH to function" +#endif +""" + def render_c_file(self, header_file_name, mode): """Render the entire generated C file contents.""" log_result_define = """#define log_result(state, result, func) \ @@ -2920,6 +3029,7 @@ def render_c_file(self, header_file_name, mode): #if DEFAULT_MAX_QTY != {self.default_max_qty} #error "The type file was generated with a different default_max_qty than this file" #endif +{self.render_smart_search_check() if self.needs_map_smart_search[mode] else ''} {log_result_define} @@ -3007,6 +3117,8 @@ def render_cmake_file(self, target_name, h_files, c_files, type_file, include_dirs = sorted(set(((Path(output_h_dir)), (Path(type_file.name).parent), *((Path(h.name).parent) for h in h_files.values())))) + add_smart_search = any(self.needs_map_smart_search[mode] for mode in ("decode", "encode")) + smart_search = f'\ntarget_compile_definitions({target_name} PUBLIC ZCBOR_MAP_SMART_SEARCH)\n' def relativify(p): try: @@ -3031,7 +3143,7 @@ def relativify(p): target_include_directories({target_name} PUBLIC {(linesep + " ").join(((str(relativify(f)) for f in include_dirs)))} ) -""" +{f'{smart_search}' if add_smart_search else ''}""" def render(self, modes, h_files, c_files, type_file, include_prefix, cmake_file=None, output_c_dir=None, output_h_dir=None): @@ -3190,6 +3302,14 @@ def parse_args(): help="""Header to be included in the comment at the top of generated files, e.g. copyright. Can be a string or a path to a file. If interpreted as a path to an existing file, the file's contents will be used.""") + code_parser.add_argument( + "--unordered-maps", required=False, action="store_true", default=False, + help="""Add support in the generated code for parsing maps with unknown element order. +When enabled, the generated code will use the zcbor_unordered_map_*() API to decode data +whenever inside a map. +zcbor detects when ZCBOR_MAP_SMART_SEARCH is needed and enable +This places restrictions on the level of ambiguity allowed between map keys in a map.""" + ) code_parser.set_defaults(process=process_code) validate_parent_parser = ArgumentParser(add_help=False) @@ -3292,7 +3412,7 @@ def process_code(args): for mode in modes: cddl_res[mode] = CodeGenerator.from_cddl( mode, cddl_contents, args.default_max_qty, mode, args.entry_types, - args.default_bit_size, short_names=args.short_names) + args.default_bit_size, unordered_maps=args.unordered_maps, short_names=args.short_names) # Parsing is done, pretty print the result. verbose_print(args.verbose, "Parsed CDDL types:")