FIX oauth2 cc authorization, tests

python-sync-actions/src/component.py

@@ -88,7 +88,8 @@ def init_component(self):
                 user_params = self._configuration.user_parameters
                 user_params = self._conf_helpers.fill_in_user_parameters(user_params, user_params)
                 # apply user parameters
-                auth_method_params = self._conf_helpers.fill_in_user_parameters(authentication.parameters, user_params)
+                auth_method_params = self._conf_helpers.fill_in_user_parameters(authentication.parameters, user_params,
+                                                                                False)
                 auth_method = AuthMethodBuilder.build(authentication.type, **auth_method_params)
         except AuthBuilderError as e:
             raise UserException(e) from e

python-sync-actions/src/configuration.py

@@ -1,6 +1,5 @@
 import dataclasses
 import json
-import re
 import time
 import urllib.parse as urlparse
 from dataclasses import dataclass, field
@@ -402,26 +401,39 @@ def _convert_bearer(cls, config_parameters: dict) -> Authentication:
 
     @classmethod
     def _convert_login(cls, config_parameters: dict) -> Authentication:
+        method_mapping = {'GET': 'GET', 'POST': 'POST', 'FORM': 'POST'}
+        helpers = ConfigHelpers()
         login_request: dict = config_parameters.get('api', {}).get("authentication", {}).get("loginRequest", {})
         api_request: dict = config_parameters.get('api', {}).get("authentication", {}).get("apiRequest", {})
         # evaluate functions and user parameters
-        login_request_eval = ConfigHelpers().fill_in_user_parameters(login_request, config_parameters.get('config'))
-        api_request_eval = ConfigHelpers().fill_in_user_parameters(api_request, config_parameters.get('config'))
+        user_parameters = build_user_parameters(config_parameters)
+        user_parameters = helpers.fill_in_user_parameters(user_parameters, user_parameters)
+        login_request_eval = helpers.fill_in_user_parameters(login_request, user_parameters)
+        # the function evaluation is left for the Auth method because of the response placeholder
+        api_request_eval = helpers.fill_in_user_parameters(api_request, user_parameters, False)
 
         if not login_request:
             raise ValueError('loginRequest configuration not found in the Login 88Authentication configuration')
 
         login_endpoint: str = login_request_eval.get('endpoint')
         login_url = urlparse.urljoin(config_parameters.get('api', {}).get('baseUrl', ''), login_endpoint)
-        method: str = login_request_eval.get('method', 'GET')
+
+        method = login_request_eval.get('method', 'GET')
+
         login_request_content: RequestContent = build_request_content(method, login_request_eval.get('params', {}))
+
+        try:
+            result_method: str = method_mapping[login_request_eval.get('method', 'GET').upper()]
+        except KeyError:
+            raise ValueError(f'Unsupported method: {login_request_eval.get("method")}')
+
         login_query_parameters: dict = login_request_content.query_parameters
         login_headers: dict = login_request_eval.get('headers', {})
         api_request_headers: dict = api_request_eval.get('headers', {})
         api_request_query_parameters: dict = api_request_eval.get('params', {})
 
         parameters = {'login_endpoint': login_url,
-                      'method': method,
+                      'method': result_method,
                       'login_query_parameters': login_query_parameters,
                       'login_headers': login_headers,
                       'login_query_body': login_request_content.body,
@@ -437,7 +449,8 @@ class ConfigHelpers:
     def __init__(self):
         self.user_functions = UserFunctions()
 
-    def fill_in_user_parameters(self, conf_objects: dict, user_param: dict):
+    def fill_in_user_parameters(self, conf_objects: dict, user_param: dict,
+                                evaluate_conf_objects_functions: bool = True):
         """
         This method replaces user parameter references via attr + parses functions inside user parameters,
         evaluates them and fills in the resulting values
@@ -462,15 +475,18 @@ def fill_in_user_parameters(self, conf_objects: dict, user_param: dict):
                 res = self.perform_custom_function(key, user_param[key], user_param)
                 user_param[key] = res
 
-                lookup_str_func = r'"value":\{[^}]*\}'
-                new_str = f'"{key}":"{res}"'
-                steps_string = re.sub(lookup_str_func, new_str, steps_string)
-
             lookup_str = '{"attr":"' + key + '"}'
             steps_string = steps_string.replace(lookup_str, '"' + str(user_param[key]) + '"')
         new_steps = json.loads(steps_string)
         non_matched = nested_lookup('attr', new_steps)
 
+        if evaluate_conf_objects_functions:
+            for key in new_steps:
+                if isinstance(new_steps[key], dict):
+                    # in case the parameter is function, validate, execute and replace value with result
+                    res = self.perform_custom_function(key, new_steps[key], user_param)
+                    new_steps[key] = res
+
         if non_matched:
             raise ValueError(
                 'Some user attributes [{}] specified in parameters '
@@ -517,9 +533,10 @@ def perform_custom_function(self, key: str, function_cfg: dict, user_params: dic
 
         elif function_cfg.get('attr'):
             return user_params[function_cfg['attr']]
+
         if not function_cfg.get('function'):
-            raise ValueError(
-                F'The user parameter {key} value is object and is not a valid function object: {function_cfg}')
+            for key in function_cfg:
+                function_cfg[key] = self.perform_custom_function(key, function_cfg[key], user_params)
 
         new_args = []
         if function_cfg.get('args'):
@@ -528,5 +545,6 @@ def perform_custom_function(self, key: str, function_cfg: dict, user_params: dic
                     arg = self.perform_custom_function(key, arg, user_params)
                 new_args.append(arg)
             function_cfg['args'] = new_args
-
+        if isinstance(function_cfg, dict) and not function_cfg.get('function'):
+            return function_cfg
         return self.user_functions.execute_function(function_cfg['function'], *function_cfg.get('args', []))

python-sync-actions/src/http_generic/auth.py

@@ -1,13 +1,14 @@
 import inspect
 import json
+import re
 from abc import ABC, abstractmethod
 from typing import Callable, Union, Dict, Literal
 from urllib.parse import urlencode
 
 import requests
 from requests.auth import AuthBase, HTTPBasicAuth
 
-from configuration import ContentType
+from configuration import ContentType, ConfigHelpers
 from placeholders_utils import get_data_from_path
 
 
@@ -192,7 +193,8 @@ def __init__(self, login_endpoint: str, method: str = 'GET',
         self.api_request_query_parameters = api_request_query_parameters or {}
 
     @classmethod
-    def _retrieve_response_placeholders(cls, request_object: dict, separator: str = '.', current_path: str = ''):
+    def _retrieve_response_placeholders(cls, request_object: dict, separator: str = '.', current_path: str = '') -> \
+            list[str]:
         """
         Recursively retreive all values that contain object with key `response` and return it's value and json path
         Args:
@@ -201,24 +203,12 @@ def _retrieve_response_placeholders(cls, request_object: dict, separator: str =
         Returns:
 
         """
-        values = {}
-        for key, value in request_object.items():
+        request_object_str = json.dumps(request_object, separators=(',', ':'))
+        lookup_str_func = r'"response":"([^"]*)"'
+        # Use re.search to find the pattern in your_string
+        matches = re.findall(lookup_str_func, request_object_str)
 
-            if isinstance(value, dict):
-                if not current_path:
-                    current_path = key
-                else:
-                    current_path = f"{current_path}{separator}{key}"
-                values.update(cls._retrieve_response_placeholders(value, separator, current_path))
-            elif key == 'response':
-                values[current_path] = value
-
-        return values
-
-        # lookup_str = '{"response":"' + key + '"}'
-        # steps_string = steps_string.replace(lookup_str, '"' + str(user_param[key]) + '"')
-
-        # new_steps = json.loads(steps_string)
+        return matches
 
     def _replace_placeholders_with_response(self, response_data: dict, source_object_params: dict) -> dict:
         """
@@ -232,7 +222,7 @@ def _replace_placeholders_with_response(self, response_data: dict, source_object
         """
         response_placeholders = self._retrieve_response_placeholders(source_object_params)
         source_object_params_str = json.dumps(source_object_params, separators=(',', ':'))
-        for path, placeholder in response_placeholders.items():
+        for placeholder in response_placeholders:
             lookup_str = '{"response":"' + placeholder + '"}'
             value_to_replace = get_data_from_path(placeholder, response_data, separator='.', strict=False)
             source_object_params_str = source_object_params_str.replace(lookup_str, '"' + value_to_replace + '"')
@@ -255,6 +245,12 @@ def login(self) -> Union[AuthBase, Callable]:
         self.api_request_headers = self._replace_placeholders_with_response(response.json(), self.api_request_headers)
         self.api_request_query_parameters = self._replace_placeholders_with_response(response.json(),
                                                                                      self.api_request_query_parameters)
+        cfg_helpers = ConfigHelpers()
+        self.api_request_headers = cfg_helpers.fill_in_user_parameters(self.api_request_headers, {},
+                                                                       True)
+        self.api_request_query_parameters = cfg_helpers.fill_in_user_parameters(self.api_request_query_parameters,
+                                                                                {},
+                                                                                True)
         return self
 
     def __call__(self, r):

python-sync-actions/tests/data_tests/test_003_oauth_cc/config.json

@@ -0,0 +1,123 @@
+{
+  "parameters": {
+    "api": {
+      "baseUrl": "https://private-834388-extractormock.apiary-mock.com/",
+      "http": {
+        "retryHeader": "Retry-After",
+        "maxRetries": 10,
+        "codes": [
+          408,
+          420,
+          429,
+          500,
+          502,
+          503,
+          504
+        ],
+        "connectTimeout": 30,
+        "requestTimeout": 300
+      },
+      "authentication": {
+        "type": "login",
+        "format": "json",
+        "loginRequest": {
+          "endpoint": "https://login-demo.curity.io/oauth/v2/oauth-token",
+          "method": "FORM",
+          "headers": {
+            "Accept": "application/json",
+            "Authorization": {
+              "function": "concat",
+              "args": [
+                "Basic ",
+                {
+                  "function": "base64_encode",
+                  "args": [
+                    {
+                      "function": "concat",
+                      "args": [
+                        {
+                          "attr": "__CLIENT_ID"
+                        },
+                        ":",
+                        {
+                          "attr": "#__CLIENT_SECRET"
+                        }
+                      ]
+                    }
+                  ]
+                }
+              ]
+            }
+          },
+          "params": {
+            "grant_type": "client_credentials",
+            "scope": "read"
+          }
+        },
+        "apiRequest": {
+          "headers": {
+            "Authorization": {
+              "function": "concat",
+              "args": [
+                "Bearer ",
+                {
+                  "response": "access_token"
+                }
+              ]
+            }
+          }
+        }
+      }
+    },
+    "config": {
+      "outputBucket": "TESTUPPER",
+      "incrementalOutput": true,
+      "jobs": [
+        {
+          "__NAME": "post",
+          "endpoint": "get",
+          "method": "GET",
+          "dataType": "buckets",
+          "dataField": {
+            "path": ".",
+            "delimiter": "."
+          },
+          "scroller": "events"
+        }
+      ],
+      "includes": {
+        "function": "date",
+        "args": [
+          "Y-m-d H:i:s",
+          {
+            "function": "strtotime",
+            "args": [
+              "-2 day",
+              {
+                "time": "currentStart"
+              }
+            ]
+          }
+        ]
+      },
+      "mappings": {},
+      "userData": {
+        "extraction_date": {
+          "function": "date",
+          "args": [
+            "Y-m-d H:i:s",
+            {
+              "time": "currentStart"
+            }
+          ]
+        }
+      },
+      "debug": true,
+      "__AUTH_METHOD": "oauth2",
+      "__CLIENT_ID": "demo-backend-client",
+      "#__CLIENT_SECRET": "MJlO3binatD9jk1"
+    },
+    "__SELECTED_JOB": "0"
+  },
+  "action": "test_request"
+}