Common Logger Initialisation Improvements (#3)

* Generic initialise to handle file handlers

* Template for pull requests

Author: kashyap-thacker

.github/pull_request_template.md

@@ -0,0 +1,23 @@
+## Jira:
+
+[SEN-4452](https://simpplr.atlassian.net/browse/SEN-4452)
+
+## Description:
+
+Please include a summary of the changes in bullet points here.
+Also add screenshots if available.
+
+## Type of change:
+
+Please delete options that are not relevant.
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+
+## Checklist (put x in bracket to check):
+
+Please do not delete options here.
+- [ ] I have performed a self-review of my code
+- [ ] Existing unit tests pass locally with my changes
+- [ ] I have written coverage for new code
+- [ ] I have updated the [Production deployment](https://docs.google.com/spreadsheets/d/1_tc_7h3yPxScUb2JvZyxG0qx8Dl36bP7Fh-86_0kI1A) sheet if this service required production changes in this sprint

python_common_logger/__init__.py

@@ -15,9 +15,14 @@
     Logger.initialise_console_logger(logger_name, service_name, level=logging.WARNING, context_config=None)
     ContextHandler.get_thread_execution_context(key='execution_context')
     ContextHandler.update_execution_context(execution_context, key='execution_context', reset=False)
+    LoggerUtils.create_stream_handler
+    LoggerUtils.create_json_formatter
+    LoggerUtils.create_simple_handler
+    LoggerUtils.create_simple_formatter
 """
 
 from .src import logger as Logger
+from .src.utils import logger as LoggerUtils
 from .src.context import context_handler as ContextHandler
 from .src.django import middleware as DjangoMiddleware
 

python_common_logger/src/logger.py

@@ -1,14 +1,16 @@
 import logging 
 import json
-import sys
+
+from logging import Handler, StreamHandler, Formatter, Filter
 
 from .context.context_handler import get_thread_execution_context
 from .context.execution_context import ExecutionContext, ExecutionContextType
 
 from .constants.logger import LoggerKeys, LoggerContextConfigKeys
 from .constants.context import ExecutionContextType
+from .utils.logger import create_json_formatter, create_stream_handler
 
-class ContextFilter(logging.Filter):
+class ContextFilter(Filter):
     """
     Log filter to extract the execution context from thread locals and populate it in the log record
     """
@@ -27,6 +29,7 @@ def filter(self, record):
 
 def initialise_console_logger(logger_name, service_name, level=logging.WARNING, context_config=None):
     """
+    @deprecated - use initialise_logger instead
     Initialises the logger with the handler, formatter and filter to log context data along with message
     in JSON format on the console.
 
@@ -36,6 +39,25 @@ def initialise_console_logger(logger_name, service_name, level=logging.WARNING,
         level (int, optional): Log level. Defaults to logging.WARNING.
         context_config (dict, optional): Context config to configure logging parameters See LoggerContextConfigKeys for list of allowed params. Defaults to None.
 
+    Returns:
+        Logger: Initialised logger
+    """
+    return initialise_logger(logger_name, service_name, level, context_config)
+
+
+def initialise_logger(logger_name, service_name, level=logging.INFO, context_config=None, use_default_json_handler=True, other_handlers: [Handler]=[], propagate_to_parent=False):
+    """
+    Initialises the logger.
+
+    Args:
+        logger_name (string): Name of the logger to be initialised
+        service_name (string): Service name that appears as the source in the logs
+        level (int, optional): Log level. Defaults to logging.INFO.
+        context_config (dict, optional): Context config to configure formatter for logging parameters. See LoggerContextConfigKeys for list of allowed params. Useless if use_default_json_handler is False. Defaults to None. 
+        use_default_json_handler (boolean, optional): Use the default JSON logger. Defaults to True.
+        other_handlers ([Handler], optional): Handlers to be attached to the logger. Defaults to [].
+        propagate_to_parent (boolean, optional): Should the log be propagated to parent. Defaults to False.
+
     Returns:
         Logger: Initialised logger
     """
@@ -44,43 +66,20 @@ def initialise_console_logger(logger_name, service_name, level=logging.WARNING,
     # Skip if already initialised. Helps preventing re-initialisation as Lambda instances share the logger instance.
     if hasattr(logger, 'initialized'):
         return logger
-    
-    # Create handlers
-    log_handler = logging.StreamHandler(sys.stdout)
-
-    log_format = {
-        "source": f"{service_name}",
-        "time": "%(asctime)s",
-        "log": {
-            "message": "%(message)s"
-        },
-        "logLevel": "%(levelname)s"
-    }
-
-    if not context_config:
-        context_config = {}
-
-    if not context_config.get(LoggerContextConfigKeys.DISABLE_CID.value):
-        log_format[LoggerKeys.CORRELATION_ID.value] = f"%({ExecutionContextType.CORRELATION_ID.value})s"
-    
-    if not context_config.get(LoggerContextConfigKeys.DISABLE_TID.value):
-        log_format[LoggerKeys.TENANT_ID.value] = f"%({ExecutionContextType.TENANT_ID.value})s"
-    
-    if not context_config.get(LoggerContextConfigKeys.DISABLE_UID.value):
-        log_format[LoggerKeys.USER_ID.value] = f"%({ExecutionContextType.USER_ID.value})s"
-    
-    # Create formatters and add it to handlers
-    log_formatter = logging.Formatter(json.dumps(log_format), datefmt='%Y-%m-%dT%H:%M:%S%z')
-    log_handler.setFormatter(log_formatter)
-    
-    # Populate Context Filter in Record
-    log_handler.addFilter(ContextFilter())
 
-    logger.addHandler(log_handler)
+    if use_default_json_handler:
+        json_formatter: Formatter = create_json_formatter(service_name, context_config)
+
+        json_handler: Handler = create_stream_handler(json_formatter, ContextFilter())
+
+        logger.addHandler(json_handler)
+    
+    for handler in other_handlers:
+        logger.addHandler(handler)
 
     logger.setLevel(level)
 
-    logger.propagate = False
+    logger.propagate = propagate_to_parent
     setattr(logger, 'initialized', True)
 
-    return logger
+    return logger

python_common_logger/src/utils/logger.py

@@ -0,0 +1,101 @@
+import logging 
+import json
+import sys
+
+from logging import Handler, StreamHandler, Formatter, Filter
+from ..constants.logger import LoggerKeys, LoggerContextConfigKeys
+from ..constants.context import ExecutionContextType
+
+DEFAULT_DATE_FORMAT='%Y-%m-%dT%H:%M:%S%z'
+
+def create_stream_handler(formatter: Formatter, filter: Filter=None) -> StreamHandler:
+    """
+    Create a custom stream handler
+
+    Args:
+        formatter (Formatter): Formatter for the handler
+        filter (Filter, optional): Filter for the Handler. Defaults to None.
+
+    Returns:
+        StreamHandler: Initialised Stream Handler
+    """
+    log_handler: StreamHandler = logging.StreamHandler(sys.stdout)
+
+    if filter:
+        log_handler.addFilter(filter)
+
+    log_handler.setFormatter(formatter)
+
+    return log_handler
+
+def create_json_formatter(service_name: str, context_config:dict=None, date_format:str=DEFAULT_DATE_FORMAT) -> Formatter:
+    """
+    Create a custom JSON Formatter
+
+    Args:
+        service_name (string): Service Name
+        context_config (dict, optional): Context data config. Defaults to None.
+        date_format (str, optional): Date format for the logs. Defaults to DEFAULT_DATE_FORMAT.
+
+    Returns:
+        Formatter: Log formatter
+    """
+    log_format = {
+        "source": f"{service_name}",
+        "time": "%(asctime)s",
+        "log": {
+            "message": "[%(filename)s:%(funcName)s:%(lineno)s] %(message)s"
+        },
+        "logLevel": "%(levelname)s"
+    }
+
+    if not context_config:
+        context_config = {}
+
+    if not context_config.get(LoggerContextConfigKeys.DISABLE_CID.value):
+        log_format[LoggerKeys.CORRELATION_ID.value] = f"%({ExecutionContextType.CORRELATION_ID.value})s"
+    
+    if not context_config.get(LoggerContextConfigKeys.DISABLE_TID.value):
+        log_format[LoggerKeys.TENANT_ID.value] = f"%({ExecutionContextType.TENANT_ID.value})s"
+    
+    if not context_config.get(LoggerContextConfigKeys.DISABLE_UID.value):
+        log_format[LoggerKeys.USER_ID.value] = f"%({ExecutionContextType.USER_ID.value})s"
+    
+    log_formatter = logging.Formatter(json.dumps(log_format), datefmt=date_format)
+
+    return log_formatter
+
+def create_simple_handler(date_format:str=DEFAULT_DATE_FORMAT) -> StreamHandler:
+    """
+    Creates a Simple Handler.
+    Can be used for dev / local testing, for better readability.
+
+    Args:
+        date_format (str, optional): Date format for the logs. Defaults to DEFAULT_DATE_FORMAT.
+
+    Returns:
+        StreamHandler: Log Handler
+    """
+    log_formatter = create_simple_formatter(date_format)
+    
+    log_handler = create_stream_handler(log_formatter)
+
+    return log_handler
+
+def create_simple_formatter(date_format:str=DEFAULT_DATE_FORMAT) -> Formatter:
+    """
+    Creates a Simple Formatter.
+    Can be used for dev / local testing, for better readability. 
+
+    Args:
+        date_format (str, optional): Date format for the logs. Defaults to DEFAULT_DATE_FORMAT.
+
+    Returns:
+        Formatter: Log formatter
+    """
+    log_format = "%(asctime)s %(levelname)s [%(filename)s:%(funcName)s:%(lineno)s] %(message)s"
+    
+    log_formatter = logging.Formatter(log_format, datefmt=date_format)
+
+    return log_formatter
+