feat(261): Adds parameter to descend into lists when scrubbing None p…

changelogs/fragments/261-scrub-params-descend-into-lists.yml

@@ -0,0 +1,3 @@
+---
+minor_changes:
+  - module_utils/core - add parameter ``descend_into_lists`` to ``scrub_none_parameters`` helper function (https://github.com/ansible-collections/amazon.aws/pull/262).

plugins/module_utils/core.py

@@ -359,13 +359,14 @@ def get_boto3_client_method_parameters(client, method_name, required=False):
     return parameters
 
 
-def scrub_none_parameters(parameters):
+def scrub_none_parameters(parameters, descend_into_lists=False):
     """
     Iterate over a dictionary removing any keys that have a None value
 
     Reference: https://github.com/ansible-collections/community.aws/issues/251
     Credit: https://medium.com/better-programming/how-to-remove-null-none-values-from-a-dictionary-in-python-1bedf1aab5e4
 
+    :param descend_into_lists: whether or not to descend in to lists to continue to remove None values
     :param parameters: parameter dict
     :return: parameter dict with all keys = None removed
     """
@@ -374,7 +375,9 @@ def scrub_none_parameters(parameters):
 
     for k, v in parameters.items():
         if isinstance(v, dict):
-            clean_parameters[k] = scrub_none_parameters(v)
+            clean_parameters[k] = scrub_none_parameters(v, descend_into_lists=descend_into_lists)
+        elif descend_into_lists and isinstance(v, list):
+            clean_parameters[k] = [scrub_none_parameters(vv, descend_into_lists=descend_into_lists) if isinstance(vv, dict) else vv for vv in v]
         elif v is not None:
             clean_parameters[k] = v
 

tests/unit/module_utils/core/test_scrub_none_parameters.py

@@ -1,56 +1,88 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from ansible_collections.amazon.aws.plugins.module_utils.core import scrub_none_parameters
 import pytest
 
+from ansible_collections.amazon.aws.plugins.module_utils.core import scrub_none_parameters
+
 scrub_none_test_data = [
-    (dict(),
-     dict()
+    (dict(),  # Input
+     dict(),  # Output with descend_into_lists=False
+     dict(),  # Output with descend_into_lists=True
+     ),
+    (dict(param1=None, param2=None),
+     dict(),
+     dict(),
      ),
     (dict(param1='something'),
-     dict(param1='something')
+     dict(param1='something'),
+     dict(param1='something'),
      ),
     (dict(param1=False),
-     dict(param1=False)
+     dict(param1=False),
+     dict(param1=False),
+     ),
+    (dict(param1=None, param2=[]),
+     dict(param2=[]),
+     dict(param2=[]),
+     ),
+    (dict(param1=None, param2=["list_value"]),
+     dict(param2=["list_value"]),
+     dict(param2=["list_value"]),
      ),
     (dict(param1='something', param2='something_else'),
-     dict(param1='something', param2='something_else')
+     dict(param1='something', param2='something_else'),
+     dict(param1='something', param2='something_else'),
      ),
     (dict(param1='something', param2=dict()),
-     dict(param1='something', param2=dict())
+     dict(param1='something', param2=dict()),
+     dict(param1='something', param2=dict()),
      ),
     (dict(param1='something', param2=None),
-     dict(param1='something')
+     dict(param1='something'),
+     dict(param1='something'),
      ),
     (dict(param1='something', param2=None, param3=None),
-     dict(param1='something')
+     dict(param1='something'),
+     dict(param1='something'),
      ),
     (dict(param1='something', param2=None, param3=None, param4='something_else'),
-     dict(param1='something', param4='something_else')
-     ),
-    (dict(param1=dict(sub_param1='something', sub_param2=None), param2=None, param3=None, param4='something_else'),
-     dict(param1=dict(sub_param1='something'), param4='something_else')
+     dict(param1='something', param4='something_else'),
+     dict(param1='something', param4='something_else'),
      ),
     (dict(param1=dict(sub_param1='something', sub_param2=dict(sub_sub_param1='another_thing')), param2=None, param3=None, param4='something_else'),
-     dict(param1=dict(sub_param1='something', sub_param2=dict(sub_sub_param1='another_thing')), param4='something_else')
+     dict(param1=dict(sub_param1='something', sub_param2=dict(sub_sub_param1='another_thing')), param4='something_else'),
+     dict(param1=dict(sub_param1='something', sub_param2=dict(sub_sub_param1='another_thing')), param4='something_else'),
      ),
     (dict(param1=dict(sub_param1='something', sub_param2=dict()), param2=None, param3=None, param4='something_else'),
-     dict(param1=dict(sub_param1='something', sub_param2=dict()), param4='something_else')
+     dict(param1=dict(sub_param1='something', sub_param2=dict()), param4='something_else'),
+     dict(param1=dict(sub_param1='something', sub_param2=dict()), param4='something_else'),
      ),
     (dict(param1=dict(sub_param1='something', sub_param2=False), param2=None, param3=None, param4='something_else'),
-     dict(param1=dict(sub_param1='something', sub_param2=False), param4='something_else')
+     dict(param1=dict(sub_param1='something', sub_param2=False), param4='something_else'),
+     dict(param1=dict(sub_param1='something', sub_param2=False), param4='something_else'),
      ),
-    (dict(param1=None, param2=None),
-     dict()
+    (dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2='my_dict_nested_in_a_list_2')], param2=[]),
+     dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2='my_dict_nested_in_a_list_2')], param2=[]),
+     dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2='my_dict_nested_in_a_list_2')], param2=[]),
+     ),
+    (dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2=None)], param2=[]),
+     dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2=None)], param2=[]),
+     dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1')], param2=[]),
+     ),
+    (dict(param1=[dict(sub_param1=[dict(sub_sub_param1=None)], sub_param2=None)], param2=[]),
+     dict(param1=[dict(sub_param1=[dict(sub_sub_param1=None)], sub_param2=None)], param2=[]),
+     dict(param1=[dict(sub_param1=[dict()])], param2=[]),
+     ),
+    (dict(param1=[dict(sub_param1=[dict(sub_sub_param1=None)], sub_param2=None)], param2=None),
+     dict(param1=[dict(sub_param1=[dict(sub_sub_param1=None)], sub_param2=None)]),
+     dict(param1=[dict(sub_param1=[dict()])]),
      ),
-    (dict(param1=None, param2=[]),
-     dict(param2=[])
-     )
 ]
 
 
-@pytest.mark.parametrize("input_params, output_params", scrub_none_test_data)
-def test_scrub_none_parameters(input_params, output_params):
-
-    assert scrub_none_parameters(input_params) == output_params
+@pytest.mark.parametrize("input_params, output_params_no_descend, output_params_descend", scrub_none_test_data)
+def test_scrub_none_parameters(input_params, output_params_no_descend, output_params_descend):
+    assert scrub_none_parameters(input_params) == output_params_no_descend
+    assert scrub_none_parameters(input_params, descend_into_lists=False) == output_params_no_descend
+    assert scrub_none_parameters(input_params, descend_into_lists=True) == output_params_descend