fix convert_custom_fields to handle lists as well as dicts

.circleci/config.yml

@@ -18,6 +18,12 @@ jobs:
           command: |
             source /usr/local/share/virtualenvs/tap-tester/bin/activate
             stitch-validate-json tap_mambu/schemas/*.json
+      - run:
+          when: always
+          name: 'Unit Tests'
+          command: |
+            source /usr/local/share/virtualenvs/tap-mambu/bin/activate
+            pytest tests/unittests
       - run:
           name: 'Integration Tests'
           command: |

setup.py

@@ -17,6 +17,7 @@
           'dev': [
               'ipdb==0.11',
               'pylint==2.5.3',
+              'pytest==6.2.4'
           ]
       },
       entry_points='''

tap_mambu/transform.py

@@ -42,25 +42,29 @@ def remove_custom_nodes(this_json):
         if not kk[:1] == '_'}
 
 
+def add_cust_field(key, record, cust_field_sets):
+    for cf_key, cf_value in record.items():
+        field = {
+            'field_set_id' : key,
+            'id' : cf_key,
+            'value' : cf_value,
+        }
+        cust_field_sets.append(field)
+
 # Convert custom fields and sets
 # Generalize/Abstract custom fields to key/value pairs
 def convert_custom_fields(this_json):
-    new_json = this_json
-    i = 0
     for record in this_json:
         cust_field_sets = []
-        for key in record:
-            if isinstance(record[key], dict):
-                if key[:1] == '_':
-                    for cf_key, cf_value in record[key].items():
-                        field = {}
-                        field['field_set_id'] = key
-                        field['id'] = cf_key
-                        field['value'] = cf_value
-                        cust_field_sets.append(field)
-        new_json[i]['custom_fields'] = cust_field_sets
-        i = i + 1
-    return new_json
+        for key, value in record.items():
+            if key.startswith('_'):
+                if isinstance(value, dict):
+                    add_cust_field(key, value, cust_field_sets)
+                elif isinstance(value, list):
+                    for element in value:
+                        add_cust_field(key, element, cust_field_sets)
+        record['custom_fields'] = cust_field_sets
+    return this_json
 
 
 # Run all transforms: denests _embedded, removes _embedded/_links, and

tests/unittests/test_transform.py

@@ -0,0 +1,35 @@
+from tap_mambu.transform import transform_json, convert_custom_fields, remove_custom_nodes
+import unittest
+
+
+class TestTransformJson(unittest.TestCase):
+
+    def test_transform_json_handles_dictionary_custom_fields(self):
+        expected = [{'custom_fields': [{'field_set_id': '_custom_1', 'id': 'id', 'value': '1'}],
+                     'id': '1'}]
+        actual = transform_json([{"_custom_1" : {"id": '1'}, "id": '1'}],
+                                "my_path")
+        self.assertEqual(expected, actual)
+
+    def test_transform_json_handles_list_custom_fields(self):
+        expected = [{'custom_fields': [{'field_set_id': '_custom_1', 'id': 'id', 'value': '1'},
+                                       {'field_set_id': '_custom_2', 'id': 'id', 'value': '2'},
+                                       {'field_set_id': '_custom_2', 'id': 'index', 'value': '0'},
+                                       {'field_set_id': '_custom_2', 'id': 'id', 'value': '3'},
+                                       {'field_set_id': '_custom_2', 'id': 'index', 'value': '1'}],
+                     'id': '1'}]
+
+        actual = transform_json([{"_custom_1" : {"id": '1'},
+                                  "_custom_2" : [{"id": '2', "index": '0'},
+                                                 {"id": '3', "index": '1'}],
+                                  "id": '1'}],
+                                "my_path")
+
+        self.assertEqual(expected, actual)
+
+    def test_transform_no_custom_fields(self):
+        expected = [{'custom_fields': [],
+                     'id': '1'}]
+        actual = transform_json([{"id": '1'}],
+                                "my_path")
+        self.assertEquals(expected, actual)