Skip to content

Commit

Permalink
TDL-21365 Update custom field handling and custom fields schemas (#102)
Browse files Browse the repository at this point in the history 
* Update custom field handling and custom fields schemas

* update tests with new custom field set format

* fix format

* update tests

---------

Co-authored-by: Leslie VanDeMark <[email protected]>
  • Loading branch information
@dsprayberry @leslievandemark
dsprayberry and leslievandemark authored Feb 23, 2023
1 parent e938004 commit 497ed49
Show file tree
Hide file tree
Showing 22 changed files with 278 additions and 695 deletions.
29 changes: 4 additions & 25 deletions tap_mambu/helpers/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,45 +59,24 @@ def convert_json(this_json):
return out


def remove_custom_nodes(this_json):
if not isinstance(this_json, (dict, list)):
return this_json
if isinstance(this_json, list):
return [remove_custom_nodes(vv) for vv in this_json]
return {kk: remove_custom_nodes(vv) for kk, vv in this_json.items()
if not kk[:1] == '_'}


def add_custom_field(key, record, custom_field_sets):
for cf_key, cf_value in record.items():
field = {
'field_set_id': key,
'id': cf_key,
'value': cf_value,
}
custom_field_sets.append(field)


# Convert custom fields and sets
# Generalize/Abstract custom fields to key/value pairs
# Move Custom Fields objects under custom_fields key
def convert_custom_fields(this_json):
for record in this_json:
cust_field_sets = []
for key, value in record.items():
if key.startswith('_'):
if isinstance(value, dict):
add_custom_field(key, value, cust_field_sets)
cust_field_sets.append({key: value})
elif isinstance(value, list):
for element in value:
add_custom_field(key, element, cust_field_sets)
cust_field_sets.append({key: value})
record['custom_fields'] = cust_field_sets
return this_json


# Run all transforms: denests _embedded, removes _embedded/_links, and
# convert camelCase to snake_case for fieldname keys.
def transform_json(this_json, path):
new_json = remove_custom_nodes(convert_custom_fields(this_json))
new_json = convert_custom_fields(this_json)
out = dict()
out[path] = new_json
transformed_json = convert_json(out)
Expand Down
46 changes: 13 additions & 33 deletions tap_mambu/helpers/schemas/branches.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,38 +197,18 @@
]
},
"custom_fields": {
"anyOf": [
{
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
"field_set_id": {
"type": [
"null",
"string"
]
},
"id": {
"type": [
"null",
"string"
]
},
"value": {
"type": [
"null",
"string"
]
}
}
}
},
{
"type": "null"
}
]
"type": [
"null",
"array"
],
"items": {
"type": [
"null",
"object"
],
"additionalProperties": true,
"properties": {}
}
}
}
}
}
46 changes: 13 additions & 33 deletions tap_mambu/helpers/schemas/centres.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,38 +143,18 @@
]
},
"custom_fields": {
"anyOf": [
{
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
"field_set_id": {
"type": [
"null",
"string"
]
},
"id": {
"type": [
"null",
"string"
]
},
"value": {
"type": [
"null",
"string"
]
}
}
}
},
{
"type": "null"
}
]
"type": [
"null",
"array"
],
"items": {
"type": [
"null",
"object"
],
"additionalProperties": true,
"properties": {}
}
}
}
}
}
46 changes: 13 additions & 33 deletions tap_mambu/helpers/schemas/clients.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,38 +457,18 @@
]
},
"custom_fields": {
"anyOf": [
{
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
"field_set_id": {
"type": [
"null",
"string"
]
},
"id": {
"type": [
"null",
"string"
]
},
"value": {
"type": [
"null",
"string"
]
}
}
}
},
{
"type": "null"
}
]
"type": [
"null",
"array"
],
"items": {
"type": [
"null",
"object"
],
"additionalProperties": true,
"properties": {}
}
}
}
}
}
46 changes: 13 additions & 33 deletions tap_mambu/helpers/schemas/communications.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,38 +131,18 @@
]
},
"custom_fields": {
"anyOf": [
{
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
"field_set_id": {
"type": [
"null",
"string"
]
},
"id": {
"type": [
"null",
"string"
]
},
"value": {
"type": [
"null",
"string"
]
}
}
}
},
{
"type": "null"
}
]
"type": [
"null",
"array"
],
"items": {
"type": [
"null",
"object"
],
"additionalProperties": true,
"properties": {}
}
}
}
}
}
46 changes: 13 additions & 33 deletions tap_mambu/helpers/schemas/credit_arrangements.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,38 +129,18 @@
]
},
"custom_fields": {
"anyOf": [
{
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
"field_set_id": {
"type": [
"null",
"string"
]
},
"id": {
"type": [
"null",
"string"
]
},
"value": {
"type": [
"null",
"string"
]
}
}
}
},
{
"type": "null"
}
]
"type": [
"null",
"array"
],
"items": {
"type": [
"null",
"object"
],
"additionalProperties": true,
"properties": {}
}
}
}
}
}
46 changes: 13 additions & 33 deletions tap_mambu/helpers/schemas/custom_field_sets.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,38 +266,18 @@
}
},
"custom_fields": {
"anyOf": [
{
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
"field_set_id": {
"type": [
"null",
"string"
]
},
"id": {
"type": [
"null",
"string"
]
},
"value": {
"type": [
"null",
"string"
]
}
}
}
},
{
"type": "null"
}
]
"type": [
"null",
"array"
],
"items": {
"type": [
"null",
"object"
],
"additionalProperties": true,
"properties": {}
}
}
}
}
}
Loading

0 comments on commit 497ed49

Please sign in to comment.