diff --git a/README.md b/README.md index 79ae6130..66c6dc15 100644 --- a/README.md +++ b/README.md @@ -424,7 +424,7 @@ usage: zcbor code [-h] -c CDDL [--no-prelude] [-v] ENTRY_TYPES [ENTRY_TYPES ...] [-d] [-e] [--time-header] [--git-sha-header] [-b {32,64}] [--include-prefix INCLUDE_PREFIX] [-s] - [--file-header FILE_HEADER] + [--file-header FILE_HEADER] [--unordered-maps] Parse a CDDL file and produce C code that validates and xcodes CBOR. The output from this script is a C file and a header file. The header file @@ -525,6 +525,13 @@ options: generated files, e.g. copyright. Can be a string or a path to a file. If interpreted as a path to an existing file, the file's contents will be used. + --unordered-maps Add support in the generated code for parsing maps + with unknown element order. When enabled, the + generated code will use the zcbor_unordered_map_*() + API to decode data whenever inside a map. zcbor + detects when ZCBOR_MAP_SMART_SEARCH is needed and + enable This places restrictions on the level of + ambiguity allowed between map keys in a map. ``` diff --git a/include/zcbor_common.h b/include/zcbor_common.h index 3cd4e7b9..9fdbd39e 100644 --- a/include/zcbor_common.h +++ b/include/zcbor_common.h @@ -340,7 +340,7 @@ void zcbor_new_state(zcbor_state_t *state_array, size_t n_states, */ int zcbor_entry_function(const uint8_t *payload, size_t payload_len, void *result, size_t *payload_len_out, zcbor_state_t *state, zcbor_decoder_t func, - size_t n_states, size_t elem_count); + size_t n_states, size_t elem_count, size_t n_flags); #ifdef ZCBOR_STOP_ON_ERROR /** Check stored error and fail if present, but only if stop_on_error is true. @@ -516,7 +516,7 @@ static inline size_t zcbor_flags_to_states(size_t num_flags) #define ZCBOR_FLAG_STATES(n_flags) zcbor_flags_to_states(n_flags) #else -#define ZCBOR_FLAG_STATES(n_flags) 0 +#define ZCBOR_FLAG_STATES(n_flags) (n_flags * 0) #endif size_t strnlen(const char *, size_t); diff --git a/samples/pet/pet.cmake b/samples/pet/pet.cmake index 2aa28dd6..e95408e4 100644 --- a/samples/pet/pet.cmake +++ b/samples/pet/pet.cmake @@ -21,3 +21,4 @@ target_include_directories(pet PUBLIC ${CMAKE_CURRENT_LIST_DIR}/../../include ${CMAKE_CURRENT_LIST_DIR}/include ) +target_compile_definitions(pet PUBLIC ZCBOR_MAP_SMART_SEARCH) diff --git a/samples/pet/src/pet_decode.c b/samples/pet/src/pet_decode.c index b059e741..f54084c2 100644 --- a/samples/pet/src/pet_decode.c +++ b/samples/pet/src/pet_decode.c @@ -20,6 +20,7 @@ #error "The type file was generated with a different default_max_qty than this file" #endif + static bool decode_Pet(zcbor_state_t *state, struct Pet *result); @@ -52,8 +53,8 @@ int cbor_decode_Pet( struct Pet *result, size_t *payload_len_out) { - zcbor_state_t states[4]; + zcbor_state_t states[4 + 0]; return zcbor_entry_function(payload, payload_len, (void *)result, payload_len_out, states, - (zcbor_decoder_t *)decode_Pet, sizeof(states) / sizeof(zcbor_state_t), 1); + (zcbor_decoder_t *)decode_Pet, sizeof(states) / sizeof(zcbor_state_t), 1, 0); } diff --git a/samples/pet/src/pet_encode.c b/samples/pet/src/pet_encode.c index 6b19d2c0..88fa7533 100644 --- a/samples/pet/src/pet_encode.c +++ b/samples/pet/src/pet_encode.c @@ -20,6 +20,7 @@ #error "The type file was generated with a different default_max_qty than this file" #endif + static bool encode_Pet(zcbor_state_t *state, const struct Pet *input); @@ -53,8 +54,8 @@ int cbor_encode_Pet( const struct Pet *input, size_t *payload_len_out) { - zcbor_state_t states[4]; + zcbor_state_t states[4 + 0]; return zcbor_entry_function(payload, payload_len, (void *)input, payload_len_out, states, - (zcbor_decoder_t *)encode_Pet, sizeof(states) / sizeof(zcbor_state_t), 1); + (zcbor_decoder_t *)encode_Pet, sizeof(states) / sizeof(zcbor_state_t), 1, 0); } diff --git a/src/zcbor_common.c b/src/zcbor_common.c index cd89db41..885007ca 100644 --- a/src/zcbor_common.c +++ b/src/zcbor_common.c @@ -285,9 +285,13 @@ size_t zcbor_header_len_ptr(const void *const value, size_t value_len) int zcbor_entry_function(const uint8_t *payload, size_t payload_len, void *result, size_t *payload_len_out, zcbor_state_t *state, zcbor_decoder_t func, - size_t n_states, size_t elem_count) + size_t n_states, size_t elem_count, size_t n_flags) { - zcbor_new_state(state, n_states, payload, payload_len, elem_count, NULL, 0); + zcbor_new_state(state, n_states, payload, payload_len, elem_count, + (uint8_t *)&state[n_states - 1 - ZCBOR_FLAG_STATES(n_flags)], + ZCBOR_FLAG_STATES(n_flags) * sizeof(zcbor_state_t)); + + state->constant_state->manually_process_elem = true; bool ret = func(state, result); diff --git a/tests/cases/serial_recovery.cddl b/tests/cases/serial_recovery.cddl index a8211a77..d25b6b47 100644 --- a/tests/cases/serial_recovery.cddl +++ b/tests/cases/serial_recovery.cddl @@ -4,13 +4,11 @@ ; SPDX-License-Identifier: Apache-2.0 ; -Member = ("image" => int) / - ("data" => bstr) / - ("len" => int) / - ("off" => int) / - ("sha" => bstr) / - (tstr => any) - Upload = { - 3*8members: Member + ? "image" => uint, + ? "len" => uint, + "off" => uint, + ? "sha" => bstr, + ? "data" => bstr, + ? "upgrade" => bool, } diff --git a/tests/cases/unordered_map.cddl b/tests/cases/unordered_map.cddl new file mode 100644 index 00000000..053ec21a --- /dev/null +++ b/tests/cases/unordered_map.cddl @@ -0,0 +1,36 @@ +; +; Copyright (c) 2023 Nordic Semiconductor ASA +; +; SPDX-License-Identifier: Apache-2.0 +; + +UnorderedMap1 = { + 1 => "one", + 2 => "two", + int => bstr, + "foo" => "bar", + bazboz, + *bstr => intlist, +} + +bazboz = ("baz" => "boz") +intlist = [*int] + +UnorderedMap2 = { + +typeUnion, + "map" => { + ?-1 => bstr, + ?-2 => bstr, + ?bool => int, + } +} + +typeUnion = type1 / type2 / typeDefault +type1 = 1 => tstr +type2 = 2 => tstr +typeDefault = int => nil + +UnorderedMap3 = { + 1*2000 uint => uint, + 1*2000 nint => nint, +} diff --git a/tests/decode/test3_simple/CMakeLists.txt b/tests/decode/test3_simple/CMakeLists.txt index 6621eef1..dc19f5ef 100644 --- a/tests/decode/test3_simple/CMakeLists.txt +++ b/tests/decode/test3_simple/CMakeLists.txt @@ -35,6 +35,7 @@ set(py_command_serial_recovery -d ${bit_arg} --short-names + --unordered-maps # Testing the --include-prefix option --include-prefix serial diff --git a/tests/decode/test3_simple/src/main.c b/tests/decode/test3_simple/src/main.c index 0dd120ac..60ddd8db 100644 --- a/tests/decode/test3_simple/src/main.c +++ b/tests/decode/test3_simple/src/main.c @@ -487,19 +487,15 @@ ZTEST(cbor_decode_test3, test_serial1) zassert_equal(ZCBOR_SUCCESS, ret, "decoding failed: %d.", ret); zassert_equal(sizeof(serial_rec_input1), decode_len, NULL); - zassert_equal(5, upload.members_count, - "expect 5 members"); - zassert_equal(Member_data_c, upload.members[0].members - .Member_choice, "expect data 1st"); - zassert_equal(Member_image_c, upload.members[1].members - .Member_choice, "expect image 2nd"); - zassert_equal(Member_len_c, upload.members[2].members - .Member_choice, "was %d\r\n", upload.members[2].members - .Member_choice); - zassert_equal(Member_off_c, upload.members[3].members - .Member_choice, "expect off 4th"); - zassert_equal(Member_sha_c, upload.members[4].members - .Member_choice, "expect sha 5th"); + zassert_true(upload.data_present, NULL); + zassert_equal(0x129, upload.data.data.len, NULL); + zassert_true(upload.image_present, NULL); + zassert_equal(0, upload.image.image, NULL); + zassert_true(upload.len_present, NULL); + zassert_equal(0x3b2c, upload.len.len, NULL); + zassert_equal(0, upload.off, NULL); + zassert_true(upload.sha_present, NULL); + zassert_equal(0x20, upload.sha.sha.len, NULL); } ZTEST(cbor_decode_test3, test_serial2) @@ -511,18 +507,15 @@ ZTEST(cbor_decode_test3, test_serial2) zassert_equal(ZCBOR_SUCCESS, ret, "decoding failed: %d.", ret); zassert_equal(sizeof(serial_rec_input2), decode_len, NULL); - zassert_equal(5, upload.members_count, - "expect 5 members"); - zassert_equal(Member_data_c, upload.members[0].members - .Member_choice, "expect data 1st"); - zassert_equal(Member_image_c, upload.members[1].members - .Member_choice, "expect image 2nd"); - zassert_equal(Member_len_c, upload.members[2].members - .Member_choice, "expect len 3rd"); - zassert_equal(Member_off_c, upload.members[3].members - .Member_choice, "expect off 4th"); - zassert_equal(Member_sha_c, upload.members[4].members - .Member_choice, "expect sha 5th"); + zassert_true(upload.data_present, NULL); + zassert_equal(0x129, upload.data.data.len, NULL); + zassert_true(upload.image_present, NULL); + zassert_equal(0, upload.image.image, NULL); + zassert_true(upload.len_present, NULL); + zassert_equal(0x2fe0, upload.len.len, NULL); + zassert_equal(0, upload.off, NULL); + zassert_true(upload.sha_present, NULL); + zassert_equal(0x20, upload.sha.sha.len, NULL); } ZTEST_SUITE(cbor_decode_test3, NULL, NULL, NULL, NULL, NULL); diff --git a/tests/decode/test9_manifest14/CMakeLists.txt b/tests/decode/test9_manifest14/CMakeLists.txt index c6c4cbdd..ec596d82 100644 --- a/tests/decode/test9_manifest14/CMakeLists.txt +++ b/tests/decode/test9_manifest14/CMakeLists.txt @@ -8,6 +8,7 @@ cmake_minimum_required(VERSION 3.13.1) find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) project(test9_manifest14) +set(MAP_SEARCH_FLAGS ON) # Because of --unordered maps include(../../cmake/test_template.cmake) if (NOT MANIFEST) @@ -30,6 +31,7 @@ set(py_command SUIT_Common_Sequence -d ${bit_arg} + --unordered-maps ) execute_process( diff --git a/tests/decode/testA_unordered_map/CMakeLists.txt b/tests/decode/testA_unordered_map/CMakeLists.txt new file mode 100644 index 00000000..c5f3e159 --- /dev/null +++ b/tests/decode/testA_unordered_map/CMakeLists.txt @@ -0,0 +1,39 @@ +# +# Copyright (c) 2021 Nordic Semiconductor ASA +# +# SPDX-License-Identifier: Apache-2.0 +# + +cmake_minimum_required(VERSION 3.13.1) + +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(test9_manifest14) +set(MAP_SEARCH_FLAGS ON) # Because of --unordered maps +set(MAP_SMART_SEARCH ON) +include(../../cmake/test_template.cmake) + +set(py_command + ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_LIST_DIR}/../../../zcbor/zcbor.py + code + -c ${CMAKE_CURRENT_LIST_DIR}/../../cases/unordered_map.cddl + --output-cmake ${PROJECT_BINARY_DIR}/unordered_map.cmake + -t + UnorderedMap1 + UnorderedMap2 + UnorderedMap3 + -d + ${bit_arg} + --unordered-maps + ) + +execute_process( + COMMAND + ${py_command} + COMMAND_ERROR_IS_FATAL ANY +) + +include(${PROJECT_BINARY_DIR}/unordered_map.cmake) + +target_link_libraries(unordered_map PRIVATE zephyr_interface) +target_link_libraries(app PRIVATE unordered_map) diff --git a/tests/decode/testA_unordered_map/prj.conf b/tests/decode/testA_unordered_map/prj.conf new file mode 100644 index 00000000..7cc649c9 --- /dev/null +++ b/tests/decode/testA_unordered_map/prj.conf @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Nordic Semiconductor ASA +# +# SPDX-License-Identifier: Apache-2.0 +# + +CONFIG_ZTEST=y +CONFIG_ZTEST_STACK_SIZE=131072 diff --git a/tests/decode/testA_unordered_map/src/main.c b/tests/decode/testA_unordered_map/src/main.c new file mode 100644 index 00000000..63d8f1a6 --- /dev/null +++ b/tests/decode/testA_unordered_map/src/main.c @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2023 Nordic Semiconductor ASA + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include "unordered_map_decode.h" +#include + + + +ZTEST(cbor_decode_testA, test_unordered_map1) +{ + const uint8_t payload_unordered_map1_1[] = { + MAP(7), + 3, 0x40, + 0x41, 0, LIST(3), 1, 2, 3, END + 2, 0x63, 't', 'w', 'o', + 0x63, 'b', 'a', 'z', 0x63, 'b', 'o', 'z', + 0x41, 2, LIST(0), END + 1, 0x63, 'o', 'n', 'e', + 0x63, 'f', 'o', 'o', 0x63, 'b', 'a', 'r', + END + }; + + const uint8_t payload_unordered_map1_2[] = { + MAP(5), + 2, 0x63, 't', 'w', 'o', + 3, 0x40, + 1, 0x63, 'o', 'n', 'e', + 0x63, 'b', 'a', 'z', 0x63, 'b', 'o', 'z', + 0x63, 'f', 'o', 'o', 0x63, 'b', 'a', 'r', + END + }; + + const uint8_t payload_unordered_map1_inv3[] = { + MAP(5), + 2, 0x63, 't', 'w', 'o', + 3, 0x40, + 1, 0x63, 'o', 'n', 'f' /* here */, + 0x63, 'b', 'a', 'z', 0x63, 'b', 'o', 'z', + 0x63, 'f', 'o', 'o', 0x63, 'b', 'a', 'r', + END + }; + + const uint8_t payload_unordered_map1_inv4[] = { + MAP(6), + 3, 0x40, + 0x63, 'b', 'a', 'z', 0x63, 'b', 'o', 'z', + 0x41, 0, LIST(3), 1, 2, 0x40 /* here */, END + 0x63, 'f', 'o', 'o', 0x63, 'b', 'a', 'r', + 1, 0x63, 'o', 'n', 'e', + 2, 0x63, 't', 'w', 'o', + END + }; + + struct UnorderedMap1 unordered_map1; + + int err = cbor_decode_UnorderedMap1(payload_unordered_map1_1, sizeof(payload_unordered_map1_1), &unordered_map1, NULL); + zassert_equal(ZCBOR_SUCCESS, err, "%s %d\n", zcbor_error_str(err), err); + zassert_equal(3, unordered_map1.UnorderedMap1_intbstr_key); + zassert_equal(0, unordered_map1.UnorderedMap1_intbstr.len); + zassert_equal(2, unordered_map1.UnorderedMap1_intlist_m_count); + zassert_equal(1, unordered_map1.UnorderedMap1_intlist_m[0].UnorderedMap1_intlist_m_key.len); + zassert_equal(2, unordered_map1.UnorderedMap1_intlist_m[0].UnorderedMap1_intlist_m_key.value[0]); + zassert_equal(0, unordered_map1.UnorderedMap1_intlist_m[0].UnorderedMap1_intlist_m.intlist_int_count); + zassert_equal(1, unordered_map1.UnorderedMap1_intlist_m[1].UnorderedMap1_intlist_m_key.len); + zassert_equal(0, unordered_map1.UnorderedMap1_intlist_m[1].UnorderedMap1_intlist_m_key.value[0]); + zassert_equal(3, unordered_map1.UnorderedMap1_intlist_m[1].UnorderedMap1_intlist_m.intlist_int_count); + zassert_equal(1, unordered_map1.UnorderedMap1_intlist_m[1].UnorderedMap1_intlist_m.intlist_int[0]); + zassert_equal(2, unordered_map1.UnorderedMap1_intlist_m[1].UnorderedMap1_intlist_m.intlist_int[1]); + zassert_equal(3, unordered_map1.UnorderedMap1_intlist_m[1].UnorderedMap1_intlist_m.intlist_int[2]); + + err = cbor_decode_UnorderedMap1(payload_unordered_map1_2, sizeof(payload_unordered_map1_2), &unordered_map1, NULL); + zassert_equal(ZCBOR_SUCCESS, err, "%s %d\n", zcbor_error_str(err), err); + zassert_equal(3, unordered_map1.UnorderedMap1_intbstr_key); + zassert_equal(0, unordered_map1.UnorderedMap1_intbstr.len); + zassert_equal(0, unordered_map1.UnorderedMap1_intlist_m_count); + + err = cbor_decode_UnorderedMap1(payload_unordered_map1_inv3, sizeof(payload_unordered_map1_inv3), &unordered_map1, NULL); + zassert_equal(ZCBOR_ERR_WRONG_VALUE, err, "%s\n", zcbor_error_str(err)); + + err = cbor_decode_UnorderedMap1(payload_unordered_map1_inv4, sizeof(payload_unordered_map1_inv4), &unordered_map1, NULL); + zassert_equal(ZCBOR_ERR_ELEMS_NOT_PROCESSED, err, "%s\n", zcbor_error_str(err)); +} + + +ZTEST(cbor_decode_testA, test_unordered_map2) +{ + const uint8_t payload_unordered_map2_1[] = { + MAP(4), + 1, 0x61, 'a', + 0x63, 'm', 'a', 'p', MAP(3), + 0x20, 0x40, + 0x21, 0x40, + 0xF4, 0x18, 100, + END + 0x21, 0xF6, + 2, 0x61, 'b', + END + }; + + const uint8_t payload_unordered_map2_2[] = { + MAP(2), + 0x63, 'm', 'a', 'p', MAP(1), + 0xF5, 0x39, 0x12, 0x34, + END + 0x22, 0xF6, + END + }; + + const uint8_t payload_unordered_map2_3[] = { + MAP(3), + 2, 0x66, 'f', 'o', 'o', 'b', 'a', 'r', + 0x63, 'm', 'a', 'p', MAP(2), + 0x21, 0x40, + 0x20, 0x40, + END + 1, 0x60, + END + }; + + const uint8_t payload_unordered_map2_inv4[] = { + MAP(4), + 1, 0x61, 'a', + 0x63, 'm', 'a', 'p', MAP(3), + 0x20, 0x40, + 0x21, 0x40, + 0xF4, 0xF6 /* here */, + END + 0x21, 0xF6, + 2, 0x61, 'b', + END + }; + + const uint8_t payload_unordered_map2_inv5[] = { + MAP(4), + 1, 0x61, 'a', + 0x62, 'm', 'a', /* here */ MAP(3), + 0x20, 0x40, + 0x21, 0x40, + 0xF4, 0x18, 100, + END + 0x21, 0xF6, + 2, 0x61, 'b', + END + }; + + struct UnorderedMap2 unordered_map2; + + int err = cbor_decode_UnorderedMap2(payload_unordered_map2_1, sizeof(payload_unordered_map2_1), &unordered_map2, NULL); + zassert_equal(ZCBOR_SUCCESS, err, "%s %d\n", zcbor_error_str(err), err); + zassert_equal(3, unordered_map2.UnorderedMap2_typeUnion_m_count); + zassert_equal(typeUnion_type1_m_c, unordered_map2.UnorderedMap2_typeUnion_m[0].typeUnion_choice); + zassert_equal(typeUnion_type2_m_c, unordered_map2.UnorderedMap2_typeUnion_m[1].typeUnion_choice); + zassert_equal(typeUnion_typeDefault_m_c, unordered_map2.UnorderedMap2_typeUnion_m[2].typeUnion_choice); + zassert_equal(1, unordered_map2.UnorderedMap2_typeUnion_m[0].typeUnion_type1_m.type1.len); + zassert_equal('a', unordered_map2.UnorderedMap2_typeUnion_m[0].typeUnion_type1_m.type1.value[0]); + zassert_equal(1, unordered_map2.UnorderedMap2_typeUnion_m[1].typeUnion_type2_m.type2.len); + zassert_equal('b', unordered_map2.UnorderedMap2_typeUnion_m[1].typeUnion_type2_m.type2.value[0]); + zassert_equal(-2, unordered_map2.UnorderedMap2_typeUnion_m[2].typeUnion_typeDefault_m.typeDefault_key); + zassert_true(unordered_map2.map_nint1bstr_present); + zassert_true(unordered_map2.map_nint2bstr_present); + zassert_true(unordered_map2.map_boolint_present); + zassert_equal(0, unordered_map2.map_nint1bstr.map_nint1bstr.len); + zassert_equal(0, unordered_map2.map_nint2bstr.map_nint2bstr.len); + zassert_false(unordered_map2.map_boolint.UnorderedMap2_map_boolint_key); + zassert_equal(100, unordered_map2.map_boolint.map_boolint); + + err = cbor_decode_UnorderedMap2(payload_unordered_map2_2, sizeof(payload_unordered_map2_2), &unordered_map2, NULL); + zassert_equal(ZCBOR_SUCCESS, err, "%s %d\n", zcbor_error_str(err), err); + zassert_equal(1, unordered_map2.UnorderedMap2_typeUnion_m_count); + zassert_equal(typeUnion_typeDefault_m_c, unordered_map2.UnorderedMap2_typeUnion_m[0].typeUnion_choice); + zassert_equal(-3, unordered_map2.UnorderedMap2_typeUnion_m[0].typeUnion_typeDefault_m.typeDefault_key); + zassert_true(unordered_map2.map_boolint_present); + zassert_true(unordered_map2.map_boolint.UnorderedMap2_map_boolint_key); + zassert_equal(-0x1235, unordered_map2.map_boolint.map_boolint); + + err = cbor_decode_UnorderedMap2(payload_unordered_map2_3, sizeof(payload_unordered_map2_3), &unordered_map2, NULL); + zassert_equal(ZCBOR_SUCCESS, err, "%s %d\n", zcbor_error_str(err), err); + zassert_equal(2, unordered_map2.UnorderedMap2_typeUnion_m_count); + zassert_equal(typeUnion_type1_m_c, unordered_map2.UnorderedMap2_typeUnion_m[0].typeUnion_choice); + zassert_equal(typeUnion_type2_m_c, unordered_map2.UnorderedMap2_typeUnion_m[1].typeUnion_choice); + zassert_equal(0, unordered_map2.UnorderedMap2_typeUnion_m[0].typeUnion_type1_m.type1.len); + zassert_equal(6, unordered_map2.UnorderedMap2_typeUnion_m[1].typeUnion_type2_m.type2.len); + zassert_mem_equal("foobar", unordered_map2.UnorderedMap2_typeUnion_m[1].typeUnion_type2_m.type2.value, 6); + zassert_true(unordered_map2.map_nint1bstr_present); + zassert_true(unordered_map2.map_nint2bstr_present); + zassert_equal(0, unordered_map2.map_nint1bstr.map_nint1bstr.len); + zassert_equal(0, unordered_map2.map_nint2bstr.map_nint2bstr.len); + + err = cbor_decode_UnorderedMap2(payload_unordered_map2_inv4, sizeof(payload_unordered_map2_inv4), &unordered_map2, NULL); + zassert_equal(ZCBOR_ERR_ELEMS_NOT_PROCESSED, err, "%s %d\n", zcbor_error_str(err), err); + + err = cbor_decode_UnorderedMap2(payload_unordered_map2_inv5, sizeof(payload_unordered_map2_inv5), &unordered_map2, NULL); + zassert_equal(ZCBOR_ERR_ELEM_NOT_FOUND, err, "%s %d\n", zcbor_error_str(err), err); +} + + +ZTEST(cbor_decode_testA, test_unordered_map3) +{ + uint8_t payload_unordered_map3_1[21000]; + + ZCBOR_STATE_E(state_e, 1, payload_unordered_map3_1, sizeof(payload_unordered_map3_1), 0); + + zassert_true(zcbor_map_start_encode(state_e, 2000)); + for (uint32_t i = 1; i <= 1000; i++) { + zassert_true(zcbor_uint32_put(state_e, i), "%d\n", i); + zassert_true(zcbor_uint32_put(state_e, i), "%d\n", i); + zassert_true(zcbor_int32_put(state_e, -i), "%d\n", -i); + zassert_true(zcbor_int32_put(state_e, -i), "%d\n", -i); + } + zassert_true(zcbor_map_end_encode(state_e, 2000)); + + struct UnorderedMap3 unordered_map3; + + int err = cbor_decode_UnorderedMap3(payload_unordered_map3_1, sizeof(payload_unordered_map3_1), &unordered_map3, NULL); + zassert_equal(ZCBOR_SUCCESS, err, "%s %d\n", zcbor_error_str(err), err); + + zassert_equal(1000, unordered_map3.UnorderedMap3_uintuint_count, "%d != %d", 1000, unordered_map3.UnorderedMap3_uintuint_count); + zassert_equal(1000, unordered_map3.UnorderedMap3_nintnint_count, "%d != %d", 1000, unordered_map3.UnorderedMap3_nintnint_count); +} + + +ZTEST_SUITE(cbor_decode_testA, NULL, NULL, NULL, NULL, NULL); diff --git a/tests/decode/testA_unordered_map/testcase.yaml b/tests/decode/testA_unordered_map/testcase.yaml new file mode 100644 index 00000000..bf977536 --- /dev/null +++ b/tests/decode/testA_unordered_map/testcase.yaml @@ -0,0 +1,4 @@ +tests: + zcbor.decode.testA_unordered_map: + platform_allow: native_posix native_posix_64 mps2_an521 qemu_malta_be + tags: zcbor decode unordered_map testA diff --git a/zcbor/zcbor.py b/zcbor/zcbor.py index d74e0690..d69cdddd 100755 --- a/zcbor/zcbor.py +++ b/zcbor/zcbor.py @@ -353,7 +353,7 @@ def generate_base_name(self): if self.type == "TSTR" and self.value is not None else None) # Name an integer by its expected value: or (f"{self.type.lower()}{abs(self.value)}" - if self.type in ["INT", "UINT", "NINT"] and self.value is not None else None) + if self.type in ["UINT", "NINT"] and self.value is not None else None) # Name a type by its type name or (next((key for key, value in self.my_types.items() if value == self), None)) # Name a control group by its name @@ -939,16 +939,18 @@ def get_value(self, instr): # Return the unparsed part of the string. return instr.strip() - def elem_has_key(self): + def is_valid_map_elem(self): """For checking whether this element has a key (i.e. that it is a valid "MAP" child) This must have some recursion since CDDL allows the key to be hidden behind layers of indirection. + This can be overridden by subclasses to further validate keys. """ - return self.key is not None\ - or (self.type == "OTHER" and self.my_types[self.value].elem_has_key())\ + ret = self.key is not None\ + or (self.type == "OTHER" and self.my_types[self.value].is_valid_map_elem())\ or (self.type in ["GROUP", "UNION"] - and (self.value and all(child.elem_has_key() for child in self.value))) + and (self.value and all(child.is_valid_map_elem() for child in self.value))) + return ret def post_validate(self): """Function for performing validations that must be done after all parsing is complete. @@ -957,13 +959,13 @@ def post_validate(self): """ # Validation of this element. if self.type in ["LIST", "MAP"]: - none_keys = [child for child in self.value if not child.elem_has_key()] - child_keys = [child for child in self.value if child not in none_keys] - if self.type == "MAP" and none_keys: + invalid_elems = [child for child in self.value if not child.is_valid_map_elem()] + child_keys = [child for child in self.value if child not in invalid_elems] + if self.type == "MAP" and invalid_elems: raise TypeError( - "Map member(s) must have key: " + str(none_keys) + " pointing to " + "Map member(s) are invalid: " + str(invalid_elems) + " pointing to " + str( - [self.my_types[elem.value] for elem in none_keys + [self.my_types[elem.value] for elem in invalid_elems if elem.type == "OTHER"])) if self.type == "LIST" and child_keys: raise TypeError( @@ -1854,11 +1856,13 @@ class CddlTypes(NamedTuple): class CodeGenerator(CddlXcoder): """Class for generating C code that encode/decodes CBOR and validates it according to the CDDL. """ - def __init__(self, mode, entry_type_names, default_bit_size, *args, **kwargs): + def __init__(self, mode, entry_type_names, default_bit_size, *args, + unordered_maps=False, **kwargs): super(CodeGenerator, self).__init__(*args, **kwargs) self.mode = mode self.entry_type_names = entry_type_names self.default_bit_size = default_bit_size + self.unordered_maps = unordered_maps @classmethod def from_cddl(cddl_class, mode, *args, **kwargs): @@ -1883,6 +1887,11 @@ def is_cbor(self): def init_args(self): return (self.mode, self.entry_type_names, self.default_bit_size, self.default_max_qty) + def init_kwargs(self): + kwargs = {"unordered_maps": self.unordered_maps} + kwargs.update(super().init_kwargs()) + return kwargs + def delegate_type_condition(self): """Whether to use the C type of the first child as this type's C type""" ret = (self.type in ["LIST", "MAP", "GROUP"] @@ -2257,7 +2266,7 @@ def single_func_prim(self, access, union_int=None, ptr_result=False): return (None, None) if self.type == "OTHER": - return self.my_types[self.value].single_func(access, union_int) + return self.my_types[self.value].single_func(access, union_int, ptr_result=ptr_result) func_name = self.single_func_prim_name(union_int, ptr_result=ptr_result) if func_name is None: @@ -2287,12 +2296,14 @@ def single_func_prim(self, access, union_int=None, ptr_result=False): max_val = self.max_value return (func_name, arg) - def single_func(self, access=None, union_int=None): + # Return the function name and arguments to call to encode/decode this element. + def single_func(self, access=None, union_int=None, ptr_result=False): """Return the function name and arguments to call to encode/decode this element.""" if self.single_func_impl_condition(): return (self.xcode_func_name(), deref_if_not_null(access or self.var_access())) else: - return self.single_func_prim(access or self.val_access(), union_int) + return self.single_func_prim(access or self.val_access(), union_int, + ptr_result=ptr_result) def repeated_single_func(self, ptr_result=False): """Return the function name and arguments to call to encode/decode the repeated @@ -2320,6 +2331,23 @@ def num_backups(self): total += 1 return total + def num_map_search_flags(self, in_map): + total_ex = 0 + total = 0 + if self.key: + total_ex += sum(self.key.num_map_search_flags(False)) + total += 1 + if self.cbor_var_condition(): + total_ex += sum(self.cbor.num_map_search_flags(False)) + if self.type in ["LIST", "MAP", "GROUP", "UNION"]: + child_in_map = in_map if self.type in ["GROUP", "UNION"] else self.type == "MAP" + total_ex += max([sum(child.num_map_search_flags(child_in_map)) for child in self.value]) + if self.type == "OTHER": + t, t_ex = self.my_types[self.value].num_map_search_flags(in_map) + total += t + total_ex += t_ex + return total * self.max_qty, total_ex + def depends_on(self): """Return a number indicating how many other elements this element depends on. @@ -2371,17 +2399,50 @@ def list_counts(self): }[self.type]()) return retval + def is_valid_map_elem(self): + if self.unordered_maps: + if (self.type == "GROUP") and (len(self.value) > 1) \ + and ((self.max_qty != 1) or (self.min_qty != 1)): + return False + return super().is_valid_map_elem() + + def elem_needs_map_smart_search(self, is_in_map): + if is_in_map \ + and self.unordered_maps \ + and ((self.max_qty > 1) or (self.key and not self.key.is_unambiguous_repeated())): + return True + + if self.type == "OTHER" \ + and (self.value not in self.entry_type_names or is_in_map) \ + and self.my_types[self.value].elem_needs_map_smart_search(is_in_map): + return True + + # Validation of child elements. + if self.type in ["MAP", "LIST", "UNION", "GROUP"]: + for child in self.value: + if child.elem_needs_map_smart_search( + self.type != "LIST" and (is_in_map or self.type == "MAP")): + return True + if self.cbor: + if self.cbor.elem_needs_map_smart_search(False): + return True + def xcode_list(self): """Return the full code needed to encode/decode a "LIST" or "MAP" element with children.""" start_func = f"zcbor_{self.type.lower()}_start_{self.mode}" end_func = f"zcbor_{self.type.lower()}_end_{self.mode}" end_func_force = f"zcbor_list_map_end_force_{self.mode}" + if self.type == "MAP" and self.mode == "decode" and self.unordered_maps: + start_func = "zcbor_unordered_map_start_decode" + end_func = "zcbor_unordered_map_end_decode" assert start_func in [ "zcbor_list_start_decode", "zcbor_list_start_encode", - "zcbor_map_start_decode", "zcbor_map_start_encode"] + "zcbor_map_start_decode", "zcbor_map_start_encode", + "zcbor_unordered_map_start_decode"] assert end_func in [ "zcbor_list_end_decode", "zcbor_list_end_encode", - "zcbor_map_end_decode", "zcbor_map_end_encode"] + "zcbor_map_end_decode", "zcbor_map_end_encode", + "zcbor_unordered_map_end_decode"] assert self.type in ["LIST", "MAP"], \ "Expected LIST or MAP type, was %s." % self.type _, max_counts = zip( @@ -2434,12 +2495,16 @@ def xcode_union(self): for i in range(1, len(child_values)): if ((not self.value[i].is_int_disambiguated()) and self.value[i - 1].simple_func_condition()): - child_values[i] = f"(zcbor_union_elem_code(state) && {child_values[i]})" - - return "(%s && (int_res = (%s), %s, int_res))" \ - % ("zcbor_union_start_code(state)", - f"{newl_ind}|| ".join(child_values), - "zcbor_union_end_code(state)") + if not self.value[i].key_var_condition() or not self.unordered_maps: + child_values[i] = f"(zcbor_union_elem_code(state) && {child_values[i]})" + + child_code = f"{newl_ind}|| ".join(child_values) + if len(self.value) > 0 \ + and (not self.value[0].key_var_condition() or not self.unordered_maps): + return f"(zcbor_union_start_code(state) "\ + + f"&& (int_res = ({child_code}), zcbor_union_end_code(state), int_res))" + else: + return f"({child_code})" else: return ternary_if_chain( self.choice_var_access(), @@ -2556,18 +2621,30 @@ def repeated_xcode(self, union_int=None): }[self.type] xcoders = [] if self.key: - xcoders.append(self.key.full_xcode(union_int)) + if self.mode == "decode" and self.unordered_maps: + func, *arguments = self.key.single_func(ptr_result=True) + x_args = xcode_args(*arguments) + xcoders.append( + f"zcbor_unordered_map_search((zcbor_{self.mode}r_t *){func}, {x_args})") + else: + xcoders.append(self.key.full_xcode(union_int)) if self.tags: xcoders.extend(self.xcode_tags()) if self.mode == "decode": xcoders.append(xcoder()) xcoders.extend(range_checks) + if self.key and self.unordered_maps: + xcoders.append("zcbor_elem_processed(state)") elif self.type == "BSTR" and self.cbor: xcoders.append(xcoder()) xcoders.extend(self.range_checks("tmp_str")) else: xcoders.extend(range_checks) xcoders.append(xcoder()) + # if self.key: + # # if self.mode == "decode" and self.unordered_maps: + # # xcoders.append("(true))") + # # xcoders.append("(zcbor_elem_processed(state), true))") return "(%s)" % ((newl_ind + "&& ").join(xcoders),) @@ -2662,6 +2739,8 @@ def __init__(self, entry_types, modes, print_time, default_max_qty, git_sha='', self.functions = dict() self.type_defs = dict() + self.needs_map_smart_search = dict() + # Sort type definitions so the typedefs will come in the correct order in the header file # and the function in the correct order in the c file. for mode in modes: @@ -2672,6 +2751,9 @@ def __init__(self, entry_types, modes, print_time, default_max_qty, git_sha='', self.functions[mode] = self.used_funcs(mode) self.type_defs[mode] = self.unique_types(mode) + self.needs_map_smart_search[mode] \ + = any(t.elem_needs_map_smart_search(False) for t in self.sorted_types[mode]) + self.version = __version__ if git_sha: @@ -2782,20 +2864,32 @@ def render_function(self, xcoder, mode): def render_entry_function(self, xcoder, mode): """Render a single entry function (API function) with signature and body.""" func_name, func_arg = (xcoder.xcode_func_name(), struct_ptr_name(mode)) + num_flag_states = "0" + num_flags = 0 + if xcoder.unordered_maps and mode == "decode": + num_flags = sum(xcoder.num_map_search_flags(False)) + num_flag_states = f"ZCBOR_FLAG_STATES({num_flags})" return f""" {xcoder.public_xcode_func_sig()} {{ - zcbor_state_t states[{xcoder.num_backups() + 2}]; + zcbor_state_t states[{xcoder.num_backups() + 2} + {num_flag_states}]; return zcbor_entry_function(payload, payload_len, (void *){func_arg}, payload_len_out, states, (zcbor_decoder_t *){func_name}, sizeof(states) / sizeof(zcbor_state_t), { - xcoder.list_counts()[1]}); + xcoder.list_counts()[1]}, {num_flags}); }}""" def render_file_header(self, line_prefix): lp = line_prefix return (f"\n{lp} " + self.file_header.replace("\n", f"\n{lp} ")).replace(" \n", "\n") + def render_smart_search_check(self): + return """ +#ifndef ZCBOR_MAP_SMART_SEARCH +#error "This file needs ZCBOR_MAP_SMART_SEARCH to function" +#endif +""" + def render_c_file(self, header_file_name, mode): """Render the entire generated C file contents.""" return f"""/*{self.render_file_header(" *")} @@ -2812,6 +2906,7 @@ def render_c_file(self, header_file_name, mode): #if DEFAULT_MAX_QTY != {self.default_max_qty} #error "The type file was generated with a different default_max_qty than this file" #endif +{self.render_smart_search_check() if self.needs_map_smart_search[mode] else ''} {linesep.join([self.render_forward_declaration(xcoder, mode) for xcoder in self.functions[mode]])} @@ -2921,6 +3016,7 @@ def relativify(p): target_include_directories({target_name} PUBLIC {(linesep + " ").join(((str(relativify(f)) for f in include_dirs)))} ) +target_compile_definitions({target_name} PUBLIC ZCBOR_MAP_SMART_SEARCH) """ def render(self, modes, h_files, c_files, type_file, include_prefix, cmake_file=None, @@ -3080,6 +3176,14 @@ def parse_args(): help="""Header to be included in the comment at the top of generated files, e.g. copyright. Can be a string or a path to a file. If interpreted as a path to an existing file, the file's contents will be used.""") + code_parser.add_argument( + "--unordered-maps", required=False, action="store_true", default=False, + help="""Add support in the generated code for parsing maps with unknown element order. +When enabled, the generated code will use the zcbor_unordered_map_*() API to decode data +whenever inside a map. +zcbor detects when ZCBOR_MAP_SMART_SEARCH is needed and enable +This places restrictions on the level of ambiguity allowed between map keys in a map.""" + ) code_parser.set_defaults(process=process_code) validate_parent_parser = ArgumentParser(add_help=False) @@ -3182,7 +3286,7 @@ def process_code(args): for mode in modes: cddl_res[mode] = CodeGenerator.from_cddl( mode, cddl_contents, args.default_max_qty, mode, args.entry_types, - args.default_bit_size, short_names=args.short_names) + args.default_bit_size, unordered_maps=args.unordered_maps, short_names=args.short_names) # Parsing is done, pretty print the result. verbose_print(args.verbose, "Parsed CDDL types:")