feat(261): Adds parameter to descend into lists when scrubbing None p…

changelogs/fragments/261-scrub-params-descend-into-lists.yml

@@ -0,0 +1,4 @@
+---
+minor_changes:
+  - module_utils/core - add parameter ``descend_into_lists`` to ``scrub_none_parameters`` helper function. Defaults to
+    True as the function has yet to be implemented anywhere.

plugins/module_utils/core.py

@@ -359,13 +359,14 @@ def get_boto3_client_method_parameters(client, method_name, required=False):
     return parameters
 
 
-def scrub_none_parameters(parameters):
+def scrub_none_parameters(parameters, descend_into_lists=True):
     """
     Iterate over a dictionary removing any keys that have a None value
 
     Reference: https://github.com/ansible-collections/community.aws/issues/251
     Credit: https://medium.com/better-programming/how-to-remove-null-none-values-from-a-dictionary-in-python-1bedf1aab5e4
 
+    :param descend_into_lists: whether or not to descend in to lists to continue to remove None values
     :param parameters: parameter dict
     :return: parameter dict with all keys = None removed
     """
@@ -375,6 +376,8 @@ def scrub_none_parameters(parameters):
     for k, v in parameters.items():
         if isinstance(v, dict):
             clean_parameters[k] = scrub_none_parameters(v)
+        elif descend_into_lists and isinstance(v, list):
+            clean_parameters[k] = [scrub_none_parameters(vv) if isinstance(vv, dict) else vv for vv in v]
         elif v is not None:
             clean_parameters[k] = v
 

tests/unit/module_utils/core/test_scrub_none_parameters.py

@@ -1,56 +1,94 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from ansible_collections.amazon.aws.plugins.module_utils.core import scrub_none_parameters
+#from ansible_collections.amazon.aws.plugins.module_utils.core import scrub_none_parameters
+from plugins.module_utils.core import scrub_none_parameters
 import pytest
-
 scrub_none_test_data = [
     (dict(),
+     True,
      dict()
      ),
     (dict(param1='something'),
+     True,
      dict(param1='something')
      ),
     (dict(param1=False),
+     True,
      dict(param1=False)
      ),
     (dict(param1='something', param2='something_else'),
+     True,
      dict(param1='something', param2='something_else')
      ),
     (dict(param1='something', param2=dict()),
+     True,
      dict(param1='something', param2=dict())
      ),
     (dict(param1='something', param2=None),
+     True,
      dict(param1='something')
      ),
     (dict(param1='something', param2=None, param3=None),
+     True,
      dict(param1='something')
      ),
     (dict(param1='something', param2=None, param3=None, param4='something_else'),
+     True,
      dict(param1='something', param4='something_else')
      ),
     (dict(param1=dict(sub_param1='something', sub_param2=None), param2=None, param3=None, param4='something_else'),
+     True,
      dict(param1=dict(sub_param1='something'), param4='something_else')
      ),
     (dict(param1=dict(sub_param1='something', sub_param2=dict(sub_sub_param1='another_thing')), param2=None, param3=None, param4='something_else'),
+     True,
      dict(param1=dict(sub_param1='something', sub_param2=dict(sub_sub_param1='another_thing')), param4='something_else')
      ),
     (dict(param1=dict(sub_param1='something', sub_param2=dict()), param2=None, param3=None, param4='something_else'),
+     True,
      dict(param1=dict(sub_param1='something', sub_param2=dict()), param4='something_else')
      ),
     (dict(param1=dict(sub_param1='something', sub_param2=False), param2=None, param3=None, param4='something_else'),
+     True,
      dict(param1=dict(sub_param1='something', sub_param2=False), param4='something_else')
      ),
     (dict(param1=None, param2=None),
+     True,
      dict()
      ),
     (dict(param1=None, param2=[]),
+     True,
      dict(param2=[])
+     ),
+    (dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2='my_dict_nested_in_a_list_2')], param2=[]),
+     True,
+     dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2='my_dict_nested_in_a_list_2')], param2=[])
+     ),
+    (dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2='my_dict_nested_in_a_list_2')], param2=[]),
+     False,
+     dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2='my_dict_nested_in_a_list_2')], param2=[])
+     ),
+    (dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2=None)], param2=[]),
+     True,
+     dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1')], param2=[])
+     ),
+    (dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2=None)], param2=[]),
+     False,
+     dict(param1=[dict(sub_param1='my_dict_nested_in_a_list_1', sub_param2=None)], param2=[])
+     ),
+    (dict(param1=[dict(sub_param1=[dict(sub_sub_param1=None)], sub_param2=None)], param2=[]),
+     True,
+     dict(param1=[dict(sub_param1=[{}])], param2=[])
+     ),
+    (dict(param1=[dict(sub_param1=[dict(sub_sub_param1=None)], sub_param2=None)], param2=None),
+     False,
+     dict(param1=[dict(sub_param1=[dict(sub_sub_param1=None)], sub_param2=None)]),
      )
 ]
 
 
-@pytest.mark.parametrize("input_params, output_params", scrub_none_test_data)
-def test_scrub_none_parameters(input_params, output_params):
+@pytest.mark.parametrize("input_params, descend_into_lists_flag, output_params", scrub_none_test_data)
+def test_scrub_none_parameters(input_params, descend_into_lists_flag, output_params):
 
-    assert scrub_none_parameters(input_params) == output_params
+    assert scrub_none_parameters(input_params, descend_into_lists_flag) == output_params