zcbor.py: Add support for unordered maps in generated code

MIGRATION_GUIDE.md

@@ -1,17 +1,8 @@
 # zcbor v. 0.9.99
 
-* `zcbor_simple_*()` functions have been removed to avoid confusion about their use.
-  They are still in the C file because they are used by other functions.
-  Instead, use the specific functions for the currently supported simple values, i.e.
-  `zcbor_bool_*()`, `zcbor_nil_*()`, and `zcbor_undefined_*()`.
-  If a removed variant is strictly needed, add your own forward declaration in your code.
-
-* Code generation naming:
-
-  * More C keywords are now capitalized to avoid naming collision.
-    You might have to capitalize some instances if your code was generated to have those names.
-
-  * A fix was made to the naming of bstr elements with a .size specifier, which might mean that these elements change name in your code when you regenerate.
+* `zcbor_new_state` and `zcbor_entry_function` are now deprecated in favor of `zcbor_state_init`and `zcbor_entry_func` respectively.
+  They still work as before, but will be removed in a few more releases.
+  The new functions take a struct as parameter to allow parameters to be added in the future without breaking compatibility.
 
 
 # zcbor v. 0.9.0

README.md

@@ -442,7 +442,7 @@ usage: zcbor code [-h] -c CDDL [--no-prelude] [-v]
                   -t ENTRY_TYPES [ENTRY_TYPES ...] [-d] [-e] [--time-header]
                   [--git-sha-header] [-b {32,64}]
                   [--include-prefix INCLUDE_PREFIX] [-s]
-                  [--file-header FILE_HEADER]
+                  [--file-header FILE_HEADER] [--unordered-maps]
 
 Parse a CDDL file and produce C code that validates and xcodes CBOR.
 The output from this script is a C file and a header file. The header file
@@ -543,6 +543,13 @@ options:
                         generated files, e.g. copyright. Can be a string or a
                         path to a file. If interpreted as a path to an
                         existing file, the file's contents will be used.
+  --unordered-maps      Add support in the generated code for parsing maps
+                        with unknown element order. When enabled, the
+                        generated code will use the zcbor_unordered_map_*()
+                        API to decode data whenever inside a map. zcbor
+                        detects when ZCBOR_MAP_SMART_SEARCH is needed and
+                        enable This places restrictions on the level of
+                        ambiguity allowed between map keys in a map.
 
 ```
 

include/zcbor_common.h

@@ -96,11 +96,13 @@ union {
 	                             processed. */
 };
 	uint8_t const *payload_bak; /**< Temporary backup of payload. */
-	size_t elem_count; /**< The current element is part of a LIST or a MAP,
-	                        and this keeps count of how many elements are
-	                        expected. This will be checked before processing
-	                        and decremented if the element is correctly
-	                        processed. */
+	size_t elem_count; /**< Keeps count of how many elements are expected/have been
+	                        encoded. This count is decremented for each element when
+	                        decoding, or incremented for each element when encoding.
+	                        When decoding, this is checked before decoding any element,
+	                        and decoding will fail if elem_count is 0.
+	                        When encoding, this value is eventually used to populate
+	                        the header of a list or map, when in canonical mode. */
 	uint8_t const *payload_end; /**< The end of the payload. This will be
 	                                 checked against payload before
 	                                 processing each element. */
@@ -323,23 +325,47 @@ bool zcbor_union_elem_code(zcbor_state_t *state);
  */
 bool zcbor_union_end_code(zcbor_state_t *state);
 
+/** Initialization parameters for zcbor_state_init() and zcbor_entry_func().
+ *
+ *  Note that members may be added to this struct, so initialize it in a way where
+ *  such members will be set to 0, either via a designated initializer (params = {...}),
+ *  or by memseting the struct to 0 before assigning the members.
+ */
+struct zcbor_state_init_params {
+	zcbor_state_t *states; // Memory used for the state, backup states, constant state, and flags.
+	size_t n_states; // Size of the states array.
+	const uint8_t *payload; // Note that in encoding, payload will be modified despite the `const`.
+	size_t payload_len; // Length in bytes of the payload buffer
+	size_t payload_len_out; // Will be written with the actual length used (encoded/decoded)
+	size_t elem_count; // The initial elem_count, i.e. how many elements are expected. Should be 0 for encoding.
+	uint8_t *flags; // Memory used for flags, when using ZCBOR_MAP_SMART_SEARCH and unordered maps. Ignored when ZCBOR_MAP_SMART_SEARCH is not defined.
+	size_t flags_bytes; // Length in bytes of the flags buffer. Or, if non-zero while flags is NULL, how much memory will be taken from the states array for use as flags.
+};
+
 /** Initialize a state with backups.
  *  As long as n_states is more than 1, one of the states in the array is used
  *  as a struct zcbor_state_constant object.
  *  If there is no struct zcbor_state_constant (n_states == 1), error codes are
  *  not available.
  *  This means that you get a state with (n_states - 2) backups.
- *  payload, payload_len, elem_count, and elem_state are used to initialize the first state.
- *  The elem_state is only needed for unordered maps, when ZCBOR_MAP_SMART_SEARCH is enabled.
+ *  payload, payload_len, elem_count, and flags are used to initialize the first state.
+ *  The flags is only needed for unordered maps, when ZCBOR_MAP_SMART_SEARCH is enabled.
  *  It is ignored otherwise.
  */
+
+void zcbor_state_init(struct zcbor_state_init_params *params);
+
+/** [DEPRECATED] Old variant of zcbor_state_init, kept for backwards compatibility. */
 void zcbor_new_state(zcbor_state_t *state_array, size_t n_states,
 		const uint8_t *payload, size_t payload_len, size_t elem_count,
-		uint8_t *elem_state, size_t elem_state_bytes);
+		uint8_t *flags, size_t flags_bytes);
 
 /** Do boilerplate entry function procedure.
  *  Initialize states, call function, and check the result.
  */
+int zcbor_entry_func(zcbor_decoder_t func, void *result, struct zcbor_state_init_params *params);
+
+/** [DEPRECATED] Old variant of zcbor_entry_func, kept for backwards compatibility. */
 int zcbor_entry_function(const uint8_t *payload, size_t payload_len,
 	void *result, size_t *payload_len_out, zcbor_state_t *state, zcbor_decoder_t func,
 	size_t n_states, size_t elem_count);
@@ -505,10 +531,10 @@ float zcbor_float16_to_32(uint16_t input);
  */
 uint16_t zcbor_float32_to_16(float input);
 
-#ifdef ZCBOR_MAP_SMART_SEARCH
 #define ZCBOR_ROUND_UP(x, align) (((x) + (align) - 1) / (align) * (align))
 #define ZCBOR_BITS_PER_BYTE 8
 
+#ifdef ZCBOR_MAP_SMART_SEARCH
 /** Calculate the number of bytes needed to hold @p num_flags 1 bit flags
  */
 static inline size_t zcbor_flags_to_bytes(size_t num_flags)
@@ -522,10 +548,15 @@ static inline size_t zcbor_flags_to_bytes(size_t num_flags)
 	(ZCBOR_ROUND_UP(num_flags, sizeof(zcbor_state_t) * ZCBOR_BITS_PER_BYTE) \
 			/ (sizeof(zcbor_state_t) * ZCBOR_BITS_PER_BYTE))
 
+#define ZCBOR_BYTES_TO_STATES(num_bytes) \
+	(ZCBOR_ROUND_UP(num_bytes, sizeof(zcbor_state_t)) / (sizeof(zcbor_state_t)))
+
+#define ZCBOR_BYTE_STATES(n_bytes) ZCBOR_BYTES_TO_STATES(n_bytes)
 #define ZCBOR_FLAG_STATES(n_flags) ZCBOR_FLAGS_TO_STATES(n_flags)
 
 #else
-#define ZCBOR_FLAG_STATES(n_flags) 0
+#define ZCBOR_BYTE_STATES(n_bytes) (n_bytes * 0)
+#define ZCBOR_FLAG_STATES(n_flags) (n_flags * 0)
 #endif
 
 size_t strnlen(const char *, size_t);

include/zcbor_decode.h

@@ -23,7 +23,7 @@ extern "C" {
  */
 
 
-/** See @ref zcbor_new_state() */
+/** [DEPRECATED] See @ref zcbor_new_state() and @ref zcbor_state_init() */
 void zcbor_new_decode_state(zcbor_state_t *state_array, size_t n_states,
 		const uint8_t *payload, size_t payload_len, size_t elem_count,
 		uint8_t *elem_state, size_t elem_state_bytes);
@@ -35,19 +35,27 @@ void zcbor_new_decode_state(zcbor_state_t *state_array, size_t n_states,
  *
  *  @param[in]  name          The name of the new state variable.
  *  @param[in]  num_backups   The number of backup slots to keep in the state.
- *  @param[in]  payload       The payload to work on.
+ *  @param[in]  payload_buf   The payload to work on.
  *  @param[in]  payload_size  The size (in bytes) of @p payload.
- *  @param[in]  elem_count    The starting elem_count (typically 1).
+ *  @param[in]  init_elem_count The starting elem_count (typically 1).
  *  @param[in]  n_flags       For use if ZCBOR_MAP_SMART_SEARCH is enabled, ignored otherwise.
  *                            The total number of unordered map search flags needed.
  *                            I.e. the largest number of elements expected in an unordered map,
  *                            including elements in nested unordered maps.
  */
-#define ZCBOR_STATE_D(name, num_backups, payload, payload_size, elem_count, n_flags) \
+#define ZCBOR_STATE_D(name, num_backups, payload_buf, payload_size, init_elem_count, n_flags) \
 zcbor_state_t name[((num_backups) + 2 + ZCBOR_FLAG_STATES(n_flags))]; \
 do { \
-	zcbor_new_decode_state(name, ZCBOR_ARRAY_SIZE(name), payload, payload_size, elem_count, \
-			(uint8_t *)&name[(num_backups) + 1], ZCBOR_FLAG_STATES(n_flags) * sizeof(zcbor_state_t)); \
+	struct zcbor_state_init_params params = { \
+		.states = name, \
+		.n_states = ZCBOR_ARRAY_SIZE(name), \
+		.payload = payload_buf, \
+		.payload_len = payload_size, \
+		.elem_count = init_elem_count, \
+		.flags = NULL, \
+		.flags_bytes = ZCBOR_ROUND_UP(n_flags, ZCBOR_BITS_PER_BYTE) / ZCBOR_BITS_PER_BYTE, \
+	}; \
+	zcbor_state_init(&params); \
 } while(0)
 
 

include/zcbor_encode.h

@@ -35,14 +35,23 @@ void zcbor_new_encode_state(zcbor_state_t *state_array, size_t n_states,
  *
  *  @param[in]  name          The name of the new state variable.
  *  @param[in]  num_backups   The number of backup slots to keep in the state.
- *  @param[in]  payload       The payload to work on.
+ *  @param[in]  payload_buf   The payload to work on.
  *  @param[in]  payload_size  The size (in bytes) of @p payload.
- *  @param[in]  elem_count    The starting elem_count (typically 1).
+ *  @param[in]  init_elem_count The starting elem_count (typically 1).
  */
-#define ZCBOR_STATE_E(name, num_backups, payload, payload_size, elem_count) \
+#define ZCBOR_STATE_E(name, num_backups, payload_buf, payload_size, init_elem_count) \
 zcbor_state_t name[((num_backups) + 2)]; \
 do { \
-	zcbor_new_encode_state(name, ZCBOR_ARRAY_SIZE(name), payload, payload_size, elem_count); \
+	struct zcbor_state_init_params params = { \
+		.states = name, \
+		.n_states = ZCBOR_ARRAY_SIZE(name), \
+		.payload = payload_buf, \
+		.payload_len = payload_size, \
+		.elem_count = init_elem_count, \
+		.flags = NULL, \
+		.flags_bytes = 0, \
+	}; \
+	zcbor_state_init(&params); \
 } while(0)
 
 

samples/pet/src/pet_decode.c

@@ -20,6 +20,7 @@
 #error "The type file was generated with a different default_max_qty than this file"
 #endif
 
+
 #define log_result(state, result, func) do { \
 	if (!result) { \
 		zcbor_trace_file(state); \
@@ -62,8 +63,22 @@ int cbor_decode_Pet(
 		struct Pet *result,
 		size_t *payload_len_out)
 {
-	zcbor_state_t states[4];
+	zcbor_state_t states[4 + 0];
+
+	struct zcbor_state_init_params params = {
+		.states = states,
+		.n_states = sizeof(states) / sizeof(zcbor_state_t),
+		.payload = payload,
+		.payload_len = payload_len,
+		.elem_count = 1,
+		.flags_bytes = 0,
+	};
+
+	int ret = zcbor_entry_func((zcbor_decoder_t *)decode_Pet, (void *)result, &params);
+
+	if (payload_len_out != NULL && ret == ZCBOR_SUCCESS) {
+		*payload_len_out = params.payload_len_out;
+	}
 
-	return zcbor_entry_function(payload, payload_len, (void *)result, payload_len_out, states,
-		(zcbor_decoder_t *)decode_Pet, sizeof(states) / sizeof(zcbor_state_t), 1);
+	return ret;
 }

samples/pet/src/pet_encode.c

@@ -20,6 +20,7 @@
 #error "The type file was generated with a different default_max_qty than this file"
 #endif
 
+
 #define log_result(state, result, func) do { \
 	if (!result) { \
 		zcbor_trace_file(state); \
@@ -56,8 +57,22 @@ int cbor_encode_Pet(
 		const struct Pet *input,
 		size_t *payload_len_out)
 {
-	zcbor_state_t states[4];
+	zcbor_state_t states[4 + 0];
+
+	struct zcbor_state_init_params params = {
+		.states = states,
+		.n_states = sizeof(states) / sizeof(zcbor_state_t),
+		.payload = payload,
+		.payload_len = payload_len,
+		.elem_count = 1,
+		.flags_bytes = 0,
+	};
+
+	int ret = zcbor_entry_func((zcbor_decoder_t *)encode_Pet, (void *)input, &params);
+
+	if (payload_len_out != NULL && ret == ZCBOR_SUCCESS) {
+		*payload_len_out = params.payload_len_out;
+	}
 
-	return zcbor_entry_function(payload, payload_len, (void *)input, payload_len_out, states,
-		(zcbor_decoder_t *)encode_Pet, sizeof(states) / sizeof(zcbor_state_t), 1);
+	return ret;
 }

scripts/update_version.py

@@ -15,15 +15,23 @@
 p_MIGRATION_GUIDE = Path(p_root, 'MIGRATION_GUIDE.md')
 p_common_h = Path(p_root, 'include', 'zcbor_common.h')
 
-def update_relnotes(p_relnotes, version, include_date=True):
+RELEASE_NOTES_boilerplate = """
+Any new bugs, requests, or missing features should be reported as [Github issues](https://github.com/NordicSemiconductor/zcbor/issues).
+
+## Improvements:
+
+## Bugfixes:
+"""
+
+
+def update_relnotes(p_relnotes, version, boilerplate="", include_date=True):
     relnotes_contents = p_relnotes.read_text(encoding="utf-8")
     relnotes_lines = relnotes_contents.splitlines()
     if version not in relnotes_lines[0]:
         new_date = f" ({datetime.today().strftime('%Y-%m-%d')})" if include_date else ""
         relnotes_new_header = f"# zcbor v. {version}{new_date}\n"
         if ".99" not in relnotes_lines[0]:
-            prev_entry = match(r"\A# zcbor.*?(?=# zcbor v.)", relnotes_contents, S).group(0)
-            relnotes_contents = prev_entry + relnotes_contents
+            relnotes_contents = relnotes_new_header + boilerplate + '\n\n' + relnotes_contents
         relnotes_contents = sub(r".*?\n", relnotes_new_header, relnotes_contents, count=1)
         p_relnotes.write_text(relnotes_contents, encoding="utf-8")
 
@@ -36,7 +44,7 @@ def update_relnotes(p_relnotes, version, include_date=True):
     (major, minor, bugfix) = version.split('.')
 
     p_VERSION.write_text(version, encoding="utf-8")
-    update_relnotes(p_RELEASE_NOTES, version)
+    update_relnotes(p_RELEASE_NOTES, version, boilerplate=RELEASE_NOTES_boilerplate)
     update_relnotes(p_MIGRATION_GUIDE, version, include_date=False)
     p_common_h_contents = p_common_h.read_text(encoding="utf-8")
     common_h_new_contents = sub(r"(#define ZCBOR_VERSION_MAJOR )\d+", f"\\g<1>{major}", p_common_h_contents)