Integrating sql commenter into otel_django_instrumentation

CHANGELOG.md

@@ -39,7 +39,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add metric instrumentation for WSGI
   ([#1128](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1128))
 - `opentelemetry-instrumentation-requests` Restoring metrics in requests
-  ([#1110](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1110)
+  ([#1110](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1110))
+- Integrated sqlcommenter plugin into opentelemetry-instrumentation-django
+  ([#896](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/896))
 
 
 ## [1.12.0rc1-0.31b0](https://github.com/open-telemetry/opentelemetry-python/releases/tag/v1.12.0rc1-0.31b0) - 2022-05-17

instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/__init__.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """
 
+API
+---
 Instrument `django`_ to trace Django applications.
 
 .. _django: https://pypi.org/project/django/
@@ -122,8 +124,68 @@ def response_hook(span, request, response):
 Example of the added span attribute,
 ``http.response.header.custom_response_header = ["<value1>,<value2>"]``
 
-API
----
+SQLCOMMENTER
+*****************************************
+You can optionally configure Django instrumentation to enable sqlcommenter which enriches
+the query with contextual information.
+
+Usage
+-----
+
+.. code:: python
+
+    from opentelemetry.instrumentation.django import DjangoInstrumentor
+
+    DjangoInstrumentor().instrument(is_sql_commentor_enabled=True)
+
+
+For example,
+::
+
+   Invoking Users().objects.all() will lead to sql query "select * from auth_users" but when SQLCommenter is enabled
+   the query will get appended with some configurable tags like "select * from auth_users /*metrics=value*/;"
+
+
+SQLCommenter Configurations
+***************************
+We can configure the tags to be appended to the sqlquery log by adding below variables to the settings.py
+
+SQLCOMMENTER_WITH_FRAMEWORK = True(Default) or False
+
+For example,
+::
+Enabling this flag will add django framework and it's version which is /*framework='django%3A2.2.3*/
+
+SQLCOMMENTER_WITH_CONTROLLER = True(Default) or False
+
+For example,
+::
+Enabling this flag will add controller name that handles the request /*controller='index'*/
+
+SQLCOMMENTER_WITH_ROUTE = True(Default) or False
+
+For example,
+::
+Enabling this flag will add url path that handles the request /*route='polls/'*/
+
+SQLCOMMENTER_WITH_APP_NAME = True(Default) or False
+
+For example,
+::
+Enabling this flag will add app name that handles the request /*app_name='polls'*/
+
+SQLCOMMENTER_WITH_OPENTELEMETRY = True(Default) or False
+
+For example,
+::
+Enabling this flag will add opentelemetry traceparent /*traceparent='00-fd720cffceba94bbf75940ff3caaf3cc-4fd1a2bdacf56388-01'*/
+
+SQLCOMMENTER_WITH_DB_DRIVER = True(Default) or False
+
+For example,
+::
+Enabling this flag will add name of the db driver /*db_driver='django.db.backends.postgresql'*/
+
 """
 
 from logging import getLogger
@@ -136,7 +198,9 @@ def response_hook(span, request, response):
 from opentelemetry.instrumentation.django.environment_variables import (
     OTEL_PYTHON_DJANGO_INSTRUMENT,
 )
-from opentelemetry.instrumentation.django.middleware import _DjangoMiddleware
+from opentelemetry.instrumentation.django.middleware.otel_middleware import (
+    _DjangoMiddleware,
+)
 from opentelemetry.instrumentation.django.package import _instruments
 from opentelemetry.instrumentation.django.version import __version__
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
@@ -166,6 +230,8 @@ class DjangoInstrumentor(BaseInstrumentor):
         [_DjangoMiddleware.__module__, _DjangoMiddleware.__qualname__]
     )
 
+    _sql_commenter_middleware = "opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware.SqlCommenter"
+
     def instrumentation_dependencies(self) -> Collection[str]:
         return _instruments
 
@@ -204,7 +270,13 @@ def _instrument(self, **kwargs):
         if isinstance(settings_middleware, tuple):
             settings_middleware = list(settings_middleware)
 
+        is_sql_commentor_enabled = kwargs.pop("is_sql_commentor_enabled", None)
+
+        if is_sql_commentor_enabled:
+            settings_middleware.insert(0, self._sql_commenter_middleware)
+
         settings_middleware.insert(0, self._opentelemetry_middleware)
+
         setattr(settings, _middleware_setting, settings_middleware)
 
     def _uninstrument(self, **kwargs):

instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/middleware/sqlcommenter_middleware.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python
+#
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from logging import getLogger
+from typing import Any, Type, TypeVar
+from urllib.parse import quote as urllib_quote
+
+# pylint: disable=no-name-in-module
+from django import conf, get_version
+from django.db import connection
+from django.db.backends.utils import CursorDebugWrapper
+
+from opentelemetry.trace.propagation.tracecontext import (
+    TraceContextTextMapPropagator,
+)
+
+_propagator = TraceContextTextMapPropagator()
+
+_django_version = get_version()
+_logger = getLogger(__name__)
+
+T = TypeVar("T")  # pylint: disable-msg=invalid-name
+
+
+class SqlCommenter:
+    """
+    Middleware to append a comment to each database query with details about
+    the framework and the execution context.
+    """
+
+    def __init__(self, get_response) -> None:
+        self.get_response = get_response
+
+    def __call__(self, request) -> Any:
+        with connection.execute_wrapper(_QueryWrapper(request)):
+            return self.get_response(request)
+
+
+class _QueryWrapper:
+    def __init__(self, request) -> None:
+        self.request = request
+
+    def __call__(self, execute: Type[T], sql, params, many, context) -> T:
+        # pylint: disable-msg=too-many-locals
+        with_framework = getattr(
+            conf.settings, "SQLCOMMENTER_WITH_FRAMEWORK", True
+        )
+        with_controller = getattr(
+            conf.settings, "SQLCOMMENTER_WITH_CONTROLLER", True
+        )
+        with_route = getattr(conf.settings, "SQLCOMMENTER_WITH_ROUTE", True)
+        with_app_name = getattr(
+            conf.settings, "SQLCOMMENTER_WITH_APP_NAME", True
+        )
+        with_opentelemetry = getattr(
+            conf.settings, "SQLCOMMENTER_WITH_OPENTELEMETRY", True
+        )
+        with_db_driver = getattr(
+            conf.settings, "SQLCOMMENTER_WITH_DB_DRIVER", True
+        )
+
+        db_driver = context["connection"].settings_dict.get("ENGINE", "")
+        resolver_match = self.request.resolver_match
+
+        sql_comment = _generate_sql_comment(
+            # Information about the controller.
+            controller=resolver_match.view_name
+            if resolver_match and with_controller
+            else None,
+            # route is the pattern that matched a request with a controller i.e. the regex
+            # See https://docs.djangoproject.com/en/stable/ref/urlresolvers/#django.urls.ResolverMatch.route
+            # getattr() because the attribute doesn't exist in Django < 2.2.
+            route=getattr(resolver_match, "route", None)
+            if resolver_match and with_route
+            else None,
+            # app_name is the application namespace for the URL pattern that matches the URL.
+            # See https://docs.djangoproject.com/en/stable/ref/urlresolvers/#django.urls.ResolverMatch.app_name
+            app_name=(resolver_match.app_name or None)
+            if resolver_match and with_app_name
+            else None,
+            # Framework centric information.
+            framework=f"django:{_django_version}" if with_framework else None,
+            # Information about the database and driver.
+            db_driver=db_driver if with_db_driver else None,
+            **_get_opentelemetry_values() if with_opentelemetry else {},
+        )
+
+        # TODO: MySQL truncates logs > 1024B so prepend comments
+        # instead of statements, if the engine is MySQL.
+        # See:
+        #  * https://github.com/basecamp/marginalia/issues/61
+        #  * https://github.com/basecamp/marginalia/pull/80
+        sql += sql_comment
+
+        # Add the query to the query log if debugging.
+        if context["cursor"].__class__ is CursorDebugWrapper:
+            context["connection"].queries_log.append(sql)
+
+        return execute(sql, params, many, context)
+
+
+def _generate_sql_comment(**meta) -> str:
+    """
+    Return a SQL comment with comma delimited key=value pairs created from
+    **meta kwargs.
+    """
+    key_value_delimiter = ","
+
+    if not meta:  # No entries added.
+        return ""
+
+    # Sort the keywords to ensure that caching works and that testing is
+    # deterministic. It eases visual inspection as well.
+    return (
+        " /*"
+        + key_value_delimiter.join(
+            f"{_url_quote(key)}={_url_quote(value)!r}"
+            for key, value in sorted(meta.items())
+            if value is not None
+        )
+        + "*/"
+    )
+
+
+def _url_quote(value) -> str:
+    if not isinstance(value, (str, bytes)):
+        return value
+    _quoted = urllib_quote(value)
+    # Since SQL uses '%' as a keyword, '%' is a by-product of url quoting
+    # e.g. foo,bar --> foo%2Cbar
+    # thus in our quoting, we need to escape it too to finally give
+    #      foo,bar --> foo%%2Cbar
+    return _quoted.replace("%", "%%")
+
+
+def _get_opentelemetry_values() -> dict or None:
+    """
+    Return the OpenTelemetry Trace and Span IDs if Span ID is set in the
+    OpenTelemetry execution context.
+    """
+    return _propagator.inject({})

instrumentation/opentelemetry-instrumentation-django/tests/test_middleware.py

@@ -102,11 +102,11 @@ def setUp(self):
         )
         self.env_patch.start()
         self.exclude_patch = patch(
-            "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._excluded_urls",
+            "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._excluded_urls",
             get_excluded_urls("DJANGO"),
         )
         self.traced_patch = patch(
-            "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._traced_request_attrs",
+            "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._traced_request_attrs",
             get_traced_request_attrs("DJANGO"),
         )
         self.exclude_patch.start()

instrumentation/opentelemetry-instrumentation-django/tests/test_middleware_asgi.py

@@ -104,11 +104,11 @@ def setUp(self):
         )
         self.env_patch.start()
         self.exclude_patch = patch(
-            "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._excluded_urls",
+            "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._excluded_urls",
             get_excluded_urls("DJANGO"),
         )
         self.traced_patch = patch(
-            "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._traced_request_attrs",
+            "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._traced_request_attrs",
             get_traced_request_attrs("DJANGO"),
         )
         self.exclude_patch.start()