TDL-16139 Support of service account authentication

.circleci/config.yml

@@ -23,7 +23,7 @@ jobs:
            source /usr/local/share/virtualenvs/tap-mixpanel/bin/activate
            source dev_env.sh
            pip install pylint
-           pylint tap_mixpanel -d "$PYLINT_DISABLE_LIST,too-many-statements,protected-access,redefined-builtin"
+           pylint tap_mixpanel -d "$PYLINT_DISABLE_LIST,too-many-statements,protected-access,redefined-builtin,too-many-instance-attributes"
       - run:
           name: 'JSON Validator'
           command: |

README.md

@@ -135,6 +135,9 @@ More details may be found in the [Mixpanel API Authentication](https://developer
    - `start_date` - the default value to use if no bookmark exists for an endpoint (rfc3339 date string)
    - `user_agent` (string, optional): Process and email for API logging purposes. Example: `tap-mixpanel <api_user_email@your_company.com>`
    - `api_secret` (string, `ABCdef123`): an API secret for each project in Mixpanel. This can be found in the Mixpanel Console, upper-right Settings (gear icon), Organization Settings > Projects and in the Access Keys section. For this tap, only the api_secret is needed (the api_key is legacy and the token is used only for uploading data). Each Mixpanel project has a different api_secret; therefore each Singer tap pipeline instance is for a single project.
+   - `service_account_username` (string, `username12`): Username of the service account.
+   - `service_account_secret` (string, `ABCdef123`): Secret of the service account.
+   - `project_id` (string, `10451202`): Id of the project which is connected to the provided service account.
    - `date_window_size` (integer, `30`): Number of days for date window looping through transactional endpoints with from_date and to_date. Default date_window_size is 30 days. Clients with large volumes of events may want to decrease this to 14, 7, or even down to 1-2 days.
    - `attribution_window` (integer, `5`): Latency minimum number of days to look-back to account for delays in attributing accurate results. [Default attribution window is 5 days](https://help.mixpanel.com/hc/en-us/articles/115004616486-Tracking-If-Users-Are-Offline).
    - `project_timezone` (string like `US/Pacific`): Time zone in which integer date times are stored. The project timezone may be found in the project settings in the Mixpanel console. [More info about timezones](https://help.mixpanel.com/hc/en-us/articles/115004547203-Manage-Timezones-for-Projects-in-Mixpanel). 

tap_mixpanel/__init__.py

@@ -17,7 +17,6 @@
 REQUEST_TIMEOUT = 300
 REQUIRED_CONFIG_KEYS = [
     "project_timezone",
-    "api_secret",
     "attribution_window",
     "start_date",
     "user_agent",
@@ -71,11 +70,19 @@ def main():
     else:
         api_domain = "mixpanel.com"
 
+    auth_type = parsed_args.config.get("auth_type")
+    if not auth_type:
+        auth_type = "api_secret"
+
     with MixpanelClient(
-        parsed_args.config["api_secret"],
+        parsed_args.config.get("api_secret"),
+        parsed_args.config.get("service_account_username"),
+        parsed_args.config.get("service_account_secret"),
+        parsed_args.config.get("project_id"),
         api_domain,
         request_timeout,
         parsed_args.config["user_agent"],
+        auth_type
     ) as client:
 
         state = {}

tap_mixpanel/client.py

@@ -135,16 +135,33 @@ class MixpanelClient:
     """
     The client class used for making REST calls to the Mixpanel API.
     """
-    def __init__(self, api_secret, api_domain, request_timeout, user_agent=None):
+    def __init__(self, api_secret, service_account_username, service_account_secret, project_id, api_domain,
+                 request_timeout, user_agent=None, auth_type='api_secret'):
         self.__api_secret = api_secret
+        self.__service_account_username = service_account_username
+        self.__service_account_secret = service_account_secret
+        self.__project_id = project_id
         self.__api_domain = api_domain
         self.__request_timeout = request_timeout
         self.__user_agent = user_agent
         self.__session = requests.Session()
+        self.__auth_type = auth_type
         self.__verified = False
+        self.auth_header = None
         self.disable_engage_endpoint = False
 
     def __enter__(self):
+        """
+        Set auth_header with provided credentials. If credentials is not provided, then raise the exception.
+        """
+        if self.__auth_type == 'api_secret' and self.__api_secret:
+            self.auth_header = f"Basic {str(base64.urlsafe_b64encode(self.__api_secret.encode('utf-8')), 'utf-8')}"
+        elif self.__service_account_username and self.__service_account_secret:
+            service_account_auth = f"{self.__service_account_username}:{self.__service_account_secret}"
+            self.auth_header = f"Basic {str(base64.urlsafe_b64encode(service_account_auth.encode('utf-8')), 'utf-8')}"
+        else:
+            raise Exception("Error: Missing api_secret or service account username/secret in tap config.json")
+
         self.__verified = self.check_access()
         return self
 
@@ -168,24 +185,33 @@ def check_access(self):
             bool: Returns true if credentials are verified.
                   (else raises Exception)
         """
-        if self.__api_secret is None:
-            raise Exception("Error: Missing api_secret in tap config.json.")
         headers = {}
+        params = {}
         # Endpoint: simple API call to return a single record (org settings) to test access
         url = f"https://{self.__api_domain}/api/2.0/engage"
         if self.__user_agent:
             headers["User-Agent"] = self.__user_agent
         headers["Accept"] = "application/json"
         headers[
             "Authorization"
-        ] = f"Basic {str(base64.urlsafe_b64encode(self.__api_secret.encode('utf-8')), 'utf-8')}"
+        ] = self.auth_header
 
+        if self.__project_id:
+            params["project_id"] = self.__project_id
         try:
             response = self.__session.get(
                 url=url,
+                params=params,
                 timeout=self.__request_timeout,  # Request timeout parameter
                 headers=headers,
             )
+
+            if response.status_code == 403:
+                LOGGER.error(
+                    "HTTP-error-code: 403, Error: User is not a member of this project: %s or this project is invalid",
+                    self.__project_id)
+                raise MixpanelForbiddenError from None
+
         except requests.exceptions.Timeout as err:
             LOGGER.error("TIMEOUT ERROR: %s", str(err))
             raise ReadTimeoutError from None
@@ -289,9 +315,15 @@ def request(self, method, url=None, path=None, params=None, json=None, **kwargs)
         if method == "POST":
             kwargs["headers"]["Content-Type"] = "application/json"
 
+        if self.__project_id:
+            if isinstance(params, dict):
+                params['project_id'] = self.__project_id
+            else:
+                params = f"{params}&project_id={self.__project_id}"
+
         kwargs["headers"][
             "Authorization"
-        ] = f"Basic {str(base64.urlsafe_b64encode(self.__api_secret.encode('utf-8')), 'utf-8')}"
+        ] = self.auth_header
         with metrics.http_request_timer(endpoint) as timer:
             response = self.perform_request(
                 method=method, url=url, params=params, json=json, **kwargs
@@ -331,6 +363,12 @@ def request_export(
         else:
             endpoint = "export"
 
+        if self.__project_id:
+            if isinstance(params, dict):
+                params['project_id'] = self.__project_id
+            else:
+                params = f"{params}&project_id={self.__project_id}"
+
         if "headers" not in kwargs:
             kwargs["headers"] = {}
 
@@ -344,7 +382,7 @@ def request_export(
 
         kwargs["headers"][
             "Authorization"
-        ] = f"Basic {str(base64.urlsafe_b64encode(self.__api_secret.encode('utf-8')), 'utf-8')}"
+        ] = self.auth_header
         with metrics.http_request_timer(endpoint) as timer:
             response = self.perform_request(
                 method=method, url=url, params=params, json=json, stream=True, **kwargs

tests/tap_tester/base.py

@@ -29,6 +29,7 @@ class TestMixPanelBase(BaseCase):
     start_date = ""
     end_date = ""
     eu_residency = True
+    service_account_authentication = False
     export_events = os.getenv("TAP_MIXPANEL_EXPORT_EVENTS")
 
     def tap_name(self):
@@ -82,6 +83,10 @@ def setUp(self):
         missing_envs = []
         if self.eu_residency:
             creds = {"api_secret": "TAP_MIXPANEL_EU_RESIDENCY_API_SECRET"}
+        elif self.service_account_authentication:
+            creds = {"service_account_username": "TAP_MIXPANEL_SERVICE_ACCOUNT_USERNAME",
+                    "service_account_secret": "TAP_MIXPANEL_SERVICE_ACCOUNT_SECRET",
+                    "project_id": "TAP_MIXPANEL_SERVICE_ACCOUNT_PROJECT_ID"}
         else:
             creds = {"api_secret": "TAP_MIXPANEL_API_SECRET"}
 
@@ -138,6 +143,10 @@ def get_credentials(self):
         credentials_dict = {}
         if self.eu_residency:
             creds = {"api_secret": "TAP_MIXPANEL_EU_RESIDENCY_API_SECRET"}
+        elif self.service_account_authentication:
+            creds = {"service_account_username": "TAP_MIXPANEL_SERVICE_ACCOUNT_USERNAME",
+                    "service_account_secret": "TAP_MIXPANEL_SERVICE_ACCOUNT_SECRET",
+                    "project_id": "TAP_MIXPANEL_SERVICE_ACCOUNT_PROJECT_ID"}
         else:
             creds = {"api_secret": "TAP_MIXPANEL_API_SECRET"}
 

tests/tap_tester/test_mixpanel_bookmark.py

@@ -135,7 +135,7 @@ def bookmark_test_run(self):
                             replication_key_value,
                             first_bookmark_value_utc,
                             msg="First sync bookmark was set incorrectly,"
-                            " a record with a greater replication-key value was synced.",
+                            "a record with a greater replication-key value was synced.",
                         )
 
                     for record in second_sync_messages:

tests/unittests/test_client.py

@@ -36,6 +36,9 @@ def test_request_with_url(self, mock_perform_request, mock_check_access):
         mock_perform_request.return_value = MockResponse()
         mock_client = client.MixpanelClient(
             api_secret="mock_api_secret",
+            service_account_username="mock_service_account_username",
+            service_account_secret="service_account_secret",
+            project_id="project_id",
             api_domain="mock_api_domain",
             request_timeout=300,
             user_agent="USER_AGENT"
@@ -61,6 +64,9 @@ def test_request_without_url(self, mock_perform_request, mock_check_access):
         mock_perform_request.return_value = MockResponse()
         mock_client = client.MixpanelClient(
             api_secret="mock_api_secret",
+            service_account_username="mock_service_account_username",
+            service_account_secret="service_account_secret",
+            project_id="project_id",
             api_domain="mock_api_domain",
             request_timeout=300,
             user_agent="USER_AGENT"
@@ -84,6 +90,9 @@ def test_request_export_with_url(self, mock_perform_request, mock_check_access):
         mock_perform_request.return_value = MockResponse()
         mock_client = client.MixpanelClient(
             api_secret="mock_api_secret",
+            service_account_username="mock_service_account_username",
+            service_account_secret="service_account_secret",
+            project_id="project_id",
             api_domain="mock_api_domain",
             request_timeout=300,
             user_agent="USER_AGENT"
@@ -119,6 +128,9 @@ def test_request_export_without_url(self, mock_perform_request, mock_check_acces
         mock_perform_request.return_value = MockResponse()
         mock_client = client.MixpanelClient(
             api_secret="mock_api_secret",
+            service_account_username="mock_service_account_username",
+            service_account_secret="service_account_secret",
+            project_id="project_id",
             api_domain="mock_api_domain",
             request_timeout=300,
             user_agent="USER_AGENT"

tests/unittests/test_error_handling.py

@@ -122,6 +122,9 @@ def test_perform_request_exception_handling(
         mock_request.return_value = mock_response
         mock_client = client.MixpanelClient(
             api_secret="mock_api_secret",
+            service_account_username="mock_service_account_username",
+            service_account_secret="service_account_secret",
+            project_id="project_id",
             api_domain="mock_api_domain",
             request_timeout=REQUEST_TIMEOUT,
         )
@@ -140,7 +143,6 @@ def test_perform_request_exception_handling(
         ["400 different timezone error", 400, mock_400_different_timezone(), client.MixpanelBadRequestError, "A validation exception has occurred. Please validate the timezone with the MixPanel UI under project settings."],
         ["400 timeout error", 400, MockResponse(400, text=timeout_400_error), client.MixpanelBadRequestError, "Timeout Error.(Please verify your credentials.)"],
         ["401 error", 401, MockResponse(401), client.MixpanelUnauthorizedError, "Invalid authorization credentials."],
-        ["403 error", 403, MockResponse(403), client.MixpanelForbiddenError, "User does not have permission to access the resource."],
         ["404 error", 404, MockResponse(404), client.MixpanelNotFoundError, "The resource you have specified cannot be found."],
         ["404 error", 404, mock_send_error(), client.MixpanelNotFoundError, "Resource not found error message from API response field 'error'."],
         ["404 error", 404, mock_send_message(), client.MixpanelNotFoundError, "Resource not found error message from API response field 'message'."],
@@ -158,8 +160,12 @@ def test_check_access_exception_handling(
         mock_request.return_value = mock_response
         mock_client = client.MixpanelClient(
             api_secret="mock_api_secret",
+            service_account_username="mock_service_account_username",
+            service_account_secret="service_account_secret",
+            project_id="project_id",
             api_domain="mock_api_domain",
             request_timeout=REQUEST_TIMEOUT,
+            auth_type="saa"
         )
         with self.assertRaises(error) as e:
             mock_client.check_access()
@@ -187,8 +193,12 @@ def test_request_with_handling_for_5xx_exception_handling(
         mock_request.return_value = mock_response
         mock_client = client.MixpanelClient(
             api_secret="mock_api_secret",
+            service_account_username="mock_service_account_username",
+            service_account_secret="service_account_secret",
+            project_id="project_id",
             api_domain="mock_api_domain",
             request_timeout=REQUEST_TIMEOUT,
+            auth_type="saa"
         )
         with self.assertRaises(error):
             mock_client.perform_request("GET")
@@ -201,8 +211,12 @@ def test_check_access_handle_timeout_error(self, mock_request, mock_sleep):
         """
         mock_client = client.MixpanelClient(
             api_secret="mock_api_secret",
+            service_account_username="mock_service_account_username",
+            service_account_secret="service_account_secret",
+            project_id="project_id",
             api_domain="mock_api_domain",
             request_timeout=REQUEST_TIMEOUT,
+            auth_type="saa"
         )
         with self.assertRaises(client.ReadTimeoutError):
             mock_client.check_access()
@@ -222,6 +236,9 @@ def test_check_access_402_exception_handling(
         mock_request.return_value = MockResponse(402)
         mock_client = client.MixpanelClient(
             api_secret="mock_api_secret",
+            service_account_username="mock_service_account_username",
+            service_account_secret="service_account_secret",
+            project_id="project_id",
             api_domain="mock_api_domain",
             request_timeout=REQUEST_TIMEOUT,
         )
@@ -243,7 +260,12 @@ def test_check_access_handle_timeout_error(self, mock_request, mock_time):
         """
         Check whether the request backoffs properly for `check_access` method for 5 times in case of Timeout error.
         """
-        mock_client = client.MixpanelClient(api_secret="mock_api_secret", api_domain="mock_api_domain", request_timeout=REQUEST_TIMEOUT)
+        mock_client = client.MixpanelClient(api_secret="mock_api_secret",
+                                            service_account_username="mock_service_account_username",
+                                            service_account_secret="service_account_secret",
+                                            project_id="project_id",
+                                            api_domain="mock_api_domain",
+                                            request_timeout=REQUEST_TIMEOUT)
         with self.assertRaises(requests.models.ProtocolError):
             mock_client.check_access()
 

tests/unittests/test_request_timeout_param_value.py

@@ -1,6 +1,5 @@
 import unittest
 from unittest import mock
-
 from parameterized import parameterized
 
 from tap_mixpanel.__init__ import main
@@ -76,6 +75,7 @@ def test_request_timeout_for_none_param_value(
             "https://mixpanel.com/api/2.0/engage",
             allow_redirects=True,
             headers=HEADER,
+            params={},
             timeout=REQUEST_TIMEOUT_DEFAULT,
         )
 
@@ -106,5 +106,6 @@ def test_request_timeout(
             "https://mixpanel.com/api/2.0/engage",
             allow_redirects=True,
             headers=HEADER,
+            params={},
             timeout=expected_value,
         )