Timezone improvement

mambu_tests/conftest.py

@@ -23,5 +23,10 @@ def retry(func):
     MultithreadedRequestsPool.shutdown = MagicMock()
 
 
+    from pytz import timezone
+    from tap_mambu.helpers import datetime_utils
+    datetime_utils._timezone = timezone("US/Pacific")
+
+
 def pytest_sessionfinish(session, exitstatus):
     _threads_original_shutdown()

mambu_tests/constants.py

@@ -6,5 +6,5 @@
                'start_date': '2021-06-01T00:00:00Z',
                'lookback_window': 30,
                'user_agent': '',
-               'page_size': '500',
+               'page_size': '200',
                'apikey_audit': 'apikey_audit_test'}

mambu_tests/helpers.py

@@ -1,4 +1,6 @@
 import datetime
+from urllib.parse import parse_qs
+
 import pytz
 
 from mock import MagicMock, Mock
@@ -48,6 +50,31 @@ def write_bookmark(self):
 class ClientMock:
     def __init__(self, page_size=100):
         self.page_size = page_size
+        self.request = MagicMock()
+
+
+class ClientWithDataMock(ClientMock):
+    def __init__(self, page_size=100, bookmark_field="creationDate",
+                 limit_field="limit", offset_field="offset", custom_data=None):
+        super(ClientWithDataMock, self).__init__(page_size=page_size)
+        self.base_url = "http://unit.test/api"
+        self.request = MagicMock()
+        self.limit_field = limit_field
+        self.offset_field = offset_field
+        self.data_to_serve = custom_data
+        if self.data_to_serve is None:
+            self.data_to_serve = [{"id": index, bookmark_field: f"2022-06-05T00:00:00.{index:06d}Z-07:00"}
+                                  for index in range(400)] + \
+                                 [{"id": index, bookmark_field: f"2022-06-06T00:00:00.{index:06d}Z-07:00"}
+                                  for index in range(400)]
+        self.request.side_effect = self.serve_request
+
+    def serve_request(self, *args, **kwargs):
+        params = kwargs.get("params")
+        split_params = parse_qs(params)
+        limit = int(split_params.get(self.limit_field, [None])[0])
+        offset = int(split_params.get(self.offset_field, [None])[0])
+        return self.data_to_serve[offset:limit+offset]
 
 
 class MultithreadedOffsetGeneratorFake(MultithreadedOffsetGenerator):

mambu_tests/multithreading/test_multithreaded_bookmark_daybyday_generator.py

@@ -1,7 +1,9 @@
 from mambu_tests.helpers import MultithreadedBookmarkDayByDayGeneratorFake
+from tap_mambu.helpers.datetime_utils import datetime_to_tz, str_to_localized_datetime
+
 
 def test_set_intermediary_bookmark_datetime():
-    mock_datetime_record = '2021-01-01T00:00:00.000000Z'
+    mock_datetime_record = datetime_to_tz(str_to_localized_datetime('2021-01-01T00:00:00.000000Z'), "UTC")
 
     generator = MultithreadedBookmarkDayByDayGeneratorFake()
 
@@ -11,30 +13,34 @@ def test_set_intermediary_bookmark_datetime():
 
     # check if the endpoint intermediary bookmark value and offset are set for a valid record with date time field
     generator.set_intermediary_bookmark(mock_datetime_record)
-    assert generator.endpoint_intermediary_bookmark_value == mock_datetime_record[:10]
+    assert generator.endpoint_intermediary_bookmark_value == mock_datetime_record.replace(hour=0, minute=0,
+                                                                                          second=0, microsecond=0)
     assert generator.endpoint_intermediary_bookmark_offset == 1
 
     # check if the endpoint intermediary bookmark value isn't set with a value lower than the already saved one
-    generator.set_intermediary_bookmark('2005-05-14T00:00:00.000000Z')
-    assert generator.endpoint_intermediary_bookmark_value == mock_datetime_record[:10]
+    generator.set_intermediary_bookmark(datetime_to_tz(str_to_localized_datetime('2005-05-14T00:00:00.000000Z'), "UTC"))
+    assert generator.endpoint_intermediary_bookmark_value == mock_datetime_record.replace(hour=0, minute=0,
+                                                                                          second=0, microsecond=0)
     assert generator.endpoint_intermediary_bookmark_offset == 1
 
     # check if the endpoint intermediary bookmark value and offset correctly set when multiple records have
     # the same bookmark field value
     for _ in range(3):
         generator.set_intermediary_bookmark(mock_datetime_record)
-    assert generator.endpoint_intermediary_bookmark_value == mock_datetime_record[:10]
+    assert generator.endpoint_intermediary_bookmark_value == mock_datetime_record.replace(hour=0, minute=0,
+                                                                                          second=0, microsecond=0)
     assert generator.endpoint_intermediary_bookmark_offset == 4
 
     # check if the endpoint intermediary bookmark value is properly set and the offset is reset
-    mock_datetime_record = '2021-11-11T00:00:00.000000Z'
+    mock_datetime_record = datetime_to_tz(str_to_localized_datetime('2021-11-11T00:00:00.000000Z'), "UTC")
     generator.set_intermediary_bookmark(mock_datetime_record)
-    assert generator.endpoint_intermediary_bookmark_value == mock_datetime_record[:10]
+    assert generator.endpoint_intermediary_bookmark_value == mock_datetime_record.replace(hour=0, minute=0,
+                                                                                          second=0, microsecond=0)
     assert generator.endpoint_intermediary_bookmark_offset == 1
 
 
 def test_set_intermediary_bookmark_date():
-    mock_date_record = '2020-01-01'
+    mock_date_record = datetime_to_tz(str_to_localized_datetime('2020-01-01'), "UTC")
 
     generator = MultithreadedBookmarkDayByDayGeneratorFake()
 
@@ -48,7 +54,7 @@ def test_set_intermediary_bookmark_date():
     assert generator.endpoint_intermediary_bookmark_offset == 1
 
     # check if the endpoint intermediary bookmark value isn't set with a value lower than the already saved one
-    generator.set_intermediary_bookmark('2005-05-14')
+    generator.set_intermediary_bookmark(datetime_to_tz(str_to_localized_datetime('2005-05-14'), "UTC"))
     assert generator.endpoint_intermediary_bookmark_value == mock_date_record
     assert generator.endpoint_intermediary_bookmark_offset == 1
 
@@ -60,7 +66,7 @@ def test_set_intermediary_bookmark_date():
     assert generator.endpoint_intermediary_bookmark_offset == 4
 
     # check if the endpoint intermediary bookmark value is properly set and the offset is reset
-    mock_date_record = '2021-11-11'
+    mock_date_record = datetime_to_tz(str_to_localized_datetime('2021-11-11'), "UTC")
     generator.set_intermediary_bookmark(mock_date_record)
     assert generator.endpoint_intermediary_bookmark_value == mock_date_record
     assert generator.endpoint_intermediary_bookmark_offset == 1

mambu_tests/multithreading/test_multithreaded_bookmark_generator.py

@@ -1,8 +1,10 @@
 import json
 import threading
 import time
+from datetime import datetime
 
 from mock import Mock, patch, call
+from pytz import timezone
 
 from mambu_tests.helpers import ClientMock, MultithreadedBookmarkGeneratorFake
 from tap_mambu.helpers.hashable_dict import HashableDict
@@ -231,34 +233,43 @@ def create_mock_future_pending():
     mock_transform_json.assert_called()
 
 
-@patch("tap_mambu.tap_generators.multithreaded_bookmark_generator.transform_datetime")
+@patch("tap_mambu.tap_generators.multithreaded_bookmark_generator.datetime_to_tz")
+@patch("tap_mambu.tap_generators.multithreaded_bookmark_generator.str_to_localized_datetime")
 @patch("tap_mambu.tap_generators.multithreaded_offset_generator."
        "MultithreadedOffsetGenerator.preprocess_record")
 @patch("tap_mambu.tap_generators.multithreaded_bookmark_generator."
        "MultithreadedBookmarkGenerator.set_intermediary_bookmark")
-def test_preprocess_record(mock_set_intermediary_bookmark, mock_offset_preprocess_record, mock_transform_datetime):
+def test_preprocess_record(mock_set_intermediary_bookmark, mock_offset_preprocess_record,
+                           mock_str_to_localized_datetime, mock_datetime_to_tz):
     mock_bookmark_field = 'test_field'
     generator = MultithreadedBookmarkGeneratorFake()
     generator.endpoint_bookmark_field = mock_bookmark_field
 
     # test the behaviour with a record that doesn't contain the bookmarked field
     mock_record = {'encoded_key': 'test'}
     mock_offset_preprocess_record.return_value = mock_record
-    mock_transform_datetime.return_value = mock_record
+    mock_datetime_to_tz.return_value = mock_record
+    mock_str_to_localized_datetime.return_value = mock_record
 
     generator.preprocess_record(mock_record)
-    mock_transform_datetime.assert_not_called()
+    mock_str_to_localized_datetime.assert_not_called()
+    mock_datetime_to_tz.assert_not_called()
     mock_set_intermediary_bookmark.assert_not_called()
 
     # test the behaviour with a record that does contain the bookmarked field
     mock_record = {'encoded_key': 'test',
-                   'test_field': 'test_value'}
+                   'test_field': '2022-01-01T00:00:00Z+03:00'}
+    datetime_in_local_tz = datetime.now(timezone("Europe/Bucharest")).replace(year=2022, month=1, day=1, hour=3,
+                                                                              minute=0, second=0, microsecond=0)
+    datetime_in_utc_tz = datetime.utcnow().replace(year=2022, month=1, day=1, hour=0, minute=0, second=0, microsecond=0)
     mock_offset_preprocess_record.return_value = mock_record
-    mock_transform_datetime.return_value = mock_record
+    mock_str_to_localized_datetime.return_value = datetime_in_local_tz
+    mock_datetime_to_tz.return_value = datetime_in_utc_tz
 
     generator.preprocess_record(mock_record)
-    mock_transform_datetime.assert_called_with(mock_record[mock_bookmark_field])
-    mock_set_intermediary_bookmark.assert_called_with(mock_record)
+    mock_str_to_localized_datetime.assert_called_with(mock_record[mock_bookmark_field])
+    mock_datetime_to_tz.assert_called_with(datetime_in_local_tz, "UTC")
+    mock_set_intermediary_bookmark.assert_called_with(datetime_in_utc_tz)
 
 
 @patch("tap_mambu.tap_generators.multithreaded_offset_generator."

mambu_tests/tap_generators/__init__.py

@@ -3,19 +3,24 @@
 from tap_mambu.tap_generators.child_generator import ChildGenerator
 from ..constants import config_json
 from tap_mambu.helpers.generator_processor_pairs import get_generator_processor_for_stream
+from ..helpers import ClientWithDataMock, ClientMock
 
 
-def setup_generator_base_test(stream_name, client_mock=None):
-    if client_mock is None:
-        client_mock = MagicMock()
-        client_mock.page_size = int(config_json.get("page_size", 500))
-        client_mock.request = MagicMock()
+def setup_generator_base_test(stream_name, client_mock=None, with_data=False, custom_data=None,
+                              offset_field="offset", limit_field="limit", bookmark_field="creationDate"):
     generator_classes, _ = get_generator_processor_for_stream(stream_name)
-
     generators = list()
     for generator_class in generator_classes:
+        client = client_mock
+        if client_mock is None:
+            client = ClientMock(int(config_json.get("page_size", 200)))
+            if with_data:
+                client = ClientWithDataMock(int(config_json.get("page_size", 200)), custom_data=custom_data,
+                                            offset_field=offset_field, limit_field=limit_field,
+                                            bookmark_field=bookmark_field)
+
         generator = generator_class(stream_name=stream_name,
-                                    client=client_mock,
+                                    client=client,
                                     config=config_json,
                                     state={"currently_syncing": stream_name},
                                     sub_type="self",

mambu_tests/tap_generators/test_activities_generator.py

@@ -23,16 +23,15 @@ def test_activities_generator_endpoint_config_init():
 
 
 def test_activities_generator_dict_unpacking():
-    generators = setup_generator_base_test("activities")
+    generators = setup_generator_base_test("activities", with_data=True,
+                                           custom_data=[{
+                                               "client": "N/A", "activity": {"id": index}}
+                                               for index in range(1000)])
 
     assert 1 == len(generators)
 
     generator = generators[0]
 
-    generator.client.request.side_effect = [[
-        {"client": "N/A", "activity": {"id": nr + page*4}}
-        for nr in range(5)] for page in range(3)] + [[] for _ in range(1000)] + [[] * 1000]
-
     for record in generator:
         assert "id" in record, "Record in activity generator were not unpacked " \
                                "according to spec! (see fetch_batch)"

mambu_tests/tap_generators/test_audit_trail_generator.py

@@ -1,14 +1,22 @@
+from datetime import datetime
+
 import mock
 from mock import MagicMock
 from copy import deepcopy
+
+from pytz import timezone
+
 from . import setup_generator_base_test
 from ..constants import config_json
 
 
-@mock.patch("tap_mambu.tap_generators.audit_trail_generator.utils")
-def test_audit_trail_generator_endpoint_config_init(utils_mock):
-    fake_lte_date = "2022-01-01T00:00:00.000000Z"
-    utils_mock.strftime.return_value = fake_lte_date
+@mock.patch("tap_mambu.tap_generators.audit_trail_generator.utc_now")
+def test_audit_trail_generator_endpoint_config_init(utc_now_mock):
+    fake_lte_date = datetime(year=2022, month=1, day=1,
+                             hour=0, minute=0, second=0,
+                             microsecond=0, tzinfo=timezone("UTC"))
+    fake_lte_date_str = "2022-01-01T00:00:00.000000Z"
+    utc_now_mock.return_value = fake_lte_date
 
     generators = setup_generator_base_test("audit_trail")
 
@@ -23,19 +31,22 @@ def test_audit_trail_generator_endpoint_config_init(utils_mock):
     assert generator.endpoint_params == {
         "sort_order": "asc",
         "occurred_at[gte]": "2021-06-01T00:00:00.000000Z",
-        "occurred_at[lte]": fake_lte_date
+        "occurred_at[lte]": fake_lte_date_str
     }
     assert generator.endpoint_bookmark_field == "occurred_at"
     assert generator.audit_trail_offset == 0
 
 
 @mock.patch("tap_mambu.tap_generators.audit_trail_generator.get_bookmark")
-@mock.patch("tap_mambu.tap_generators.audit_trail_generator.utils")
-def test_audit_trail_generator_bookmark(utils_mock, get_bookmark_mock):
-    fake_gte_date = "2021-09-01T00:00:00.000000Z"
-    fake_lte_date = "2022-01-01T00:00:00.000000Z"
-    get_bookmark_mock.return_value = [fake_gte_date, 2]
-    utils_mock.strftime.return_value = fake_lte_date
+@mock.patch("tap_mambu.tap_generators.audit_trail_generator.utc_now")
+def test_audit_trail_generator_bookmark(utc_now_mock, get_bookmark_mock):
+    fake_gte_date_str = "2021-09-01T00:00:00.000000Z"
+    get_bookmark_mock.return_value = [fake_gte_date_str, 2]
+    fake_lte_date = datetime(year=2022, month=1, day=1,
+                             hour=0, minute=0, second=0,
+                             microsecond=0, tzinfo=timezone("UTC"))
+    fake_lte_date_str = "2022-01-01T00:00:00.000000Z"
+    utc_now_mock.return_value = fake_lte_date
 
     generators = setup_generator_base_test("audit_trail")
 
@@ -46,8 +57,8 @@ def test_audit_trail_generator_bookmark(utils_mock, get_bookmark_mock):
     assert generator.audit_trail_offset == 2
     assert generator.endpoint_params == {
         "sort_order": "asc",
-        "occurred_at[gte]": fake_gte_date,
-        "occurred_at[lte]": fake_lte_date
+        "occurred_at[gte]": fake_gte_date_str,
+        "occurred_at[lte]": fake_lte_date_str
     }
 
 

mambu_tests/tap_generators/test_clients_generator.py

@@ -18,6 +18,6 @@ def test_clients_generator():
             {
                 "field": "lastModifiedDate",
                 "operator": "AFTER",
-                "value": '2021-06-01'
+                "value": '2021-05-31T17:00:00.000000Z'
             }
         ]

mambu_tests/tap_generators/test_deposit_transactions_generator.py

@@ -18,6 +18,6 @@ def test_deposit_transactions_generator():
             {
                 "field": "creationDate",
                 "operator": "AFTER",
-                "value": '2021-06-01'
+                "value": '2021-06-01T00:00:00.000000Z'
             }
         ]