Merge branch 'improve-loglevels' into 'main'

Improve logs for unmapped properties and allow users to set tap LOGLEVEL

See merge request meltano/sdk!300

singer_sdk/helpers/_typing.py

@@ -5,7 +5,7 @@
 import logging
 from enum import Enum
 from functools import lru_cache
-from typing import Any, Dict, List, Optional, cast
+from typing import Any, Dict, List, Optional, Tuple, cast
 
 import pendulum
 
@@ -173,11 +173,11 @@ def is_string_type(property_schema: dict) -> Optional[bool]:
 
 
 @lru_cache()
-def _warn_unmapped_property(
-    stream_name: str, property_name: str, logger: logging.Logger
+def _warn_unmapped_properties(
+    stream_name: str, property_names: Tuple[str], logger: logging.Logger
 ):
-    logger.warning(
-        f"Property '{property_name}' was present in the '{stream_name}' stream but "
+    logger.info(
+        f"Properties {property_names} were present in the '{stream_name}' stream but "
         "not found in catalog schema. Ignoring."
     )
 
@@ -191,9 +191,10 @@ def conform_record_data_types(  # noqa: C901
     warning will be logged exactly once per unmapped property name.
     """
     rec: Dict[str, Any] = {}
+    unmapped_properties: List[str] = []
     for property_name, elem in row.items():
         if property_name not in schema["properties"]:
-            _warn_unmapped_property(stream_name, property_name, logger)
+            unmapped_properties.append(property_name)
             continue
 
         property_schema = schema["properties"][property_name]
@@ -226,4 +227,5 @@ def conform_record_data_types(  # noqa: C901
             rec[property_name] = boolean_representation
         else:
             rec[property_name] = elem
+    _warn_unmapped_properties(stream_name, tuple(unmapped_properties), logger)
     return rec

singer_sdk/plugin_base.py

@@ -3,6 +3,7 @@
 import abc
 import json
 import logging
+import os
 from collections import OrderedDict
 from pathlib import PurePath
 from types import MappingProxyType
@@ -60,7 +61,19 @@ def logger(cls) -> logging.Logger:
         Returns:
             Plugin logger.
         """
-        return logging.getLogger(cls.name)
+        # Get the level from <PLUGIN_NAME>_LOGLEVEL or LOGLEVEL environment variables
+        LOGLEVEL = (
+            os.environ.get(f"{cls.name.upper()}_LOGLEVEL")
+            or os.environ.get("LOGLEVEL")
+            or "INFO"
+        ).upper()
+
+        assert (
+            LOGLEVEL in logging._levelToName.values()
+        ), f"Invalid LOGLEVEL configuration: {LOGLEVEL}"
+        logger = logging.getLogger(cls.name)
+        logger.setLevel(LOGLEVEL)
+        return logger
 
     # Constructor