Merge branch 'main' of github.com:open-telemetry/opentelemetry-python-contrib into middleware-always-set-status-code-on-duration-attrs

danielhochman · danielhochman · commit 6c65c4a3f43a · 2024-06-24T14:43:07.000-05:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,15 +9,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - `opentelemetry-instrumentation-aws-lambda` Bugfix: AWS Lambda event source key incorrect for SNS in instrumentation library. 
   ([#2612](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2612))
+- `opentelemetry-instrumentation-system-metrics` Permit to use psutil 6.0+.
+  ([#2630](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2630))
 - `opentelemetry-instrumentation-asgi` Bugfix: Middleware did not set status code attribute on duration metrics for non-recording spans. 
   ([#2627](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2627))
 
+
 ### Added
 
 - `opentelemetry-instrumentation-pyramid` Record exceptions raised when serving a request
   ([#2622](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2622))
 - `opentelemetry-sdk-extension-aws` Add AwsXrayLambdaPropagator
   ([#2573](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2573))
+- `opentelemetry-instrumentation-confluent-kafka` Add support for version 2.4.0 of confluent_kafka
+  ([#2616](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2616))
 
 ### Breaking changes
 
@@ -44,13 +49,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   ([#2300](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2300))
 - Rename AwsLambdaInstrumentor span attributes `faas.id` to `cloud.resource_id`, `faas.execution` to `faas.invocation_id`
   ([#2372](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2372))
-- Drop support for instrumenting elasticsearch client < 6`
+- Drop support for instrumenting elasticsearch client < 6
   ([#2422](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2422))
 - `opentelemetry-instrumentation-wsgi` Add `http.method` to `span.name`
   ([#2425](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2425))
 - `opentelemetry-instrumentation-flask` Add `http.method` to `span.name`
   ([#2454](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2454))
-- ASGI, FastAPI, Starlette: provide both send and receive hooks with `scope` and `message` for internal spans ([#2546](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2546))
+- Record repeated HTTP headers in lists, rather than a comma separate strings for ASGI based web frameworks
+  ([#2361](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2361))
+- ASGI, FastAPI, Starlette: provide both send and receive hooks with `scope` and `message` for internal spans
+- ([#2546](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2546))
 
 ### Added
 
diff --git a/docs-requirements.txt b/docs-requirements.txt
@@ -27,7 +27,7 @@ botocore~=1.0
 boto3~=1.0
 cassandra-driver~=3.25
 celery>=4.0
-confluent-kafka>= 1.8.2,<= 2.3.0
+confluent-kafka>= 1.8.2,<= 2.4.0
 elasticsearch>=6.0,<9.0
 flask~=2.0
 falcon~=2.0
diff --git a/instrumentation/README.md b/instrumentation/README.md
@@ -14,7 +14,7 @@
 | [opentelemetry-instrumentation-botocore](./opentelemetry-instrumentation-botocore) | botocore ~= 1.0 | No | experimental
 | [opentelemetry-instrumentation-cassandra](./opentelemetry-instrumentation-cassandra) | cassandra-driver ~= 3.25,scylla-driver ~= 3.25 | No | experimental
 | [opentelemetry-instrumentation-celery](./opentelemetry-instrumentation-celery) | celery >= 4.0, < 6.0 | No | experimental
-| [opentelemetry-instrumentation-confluent-kafka](./opentelemetry-instrumentation-confluent-kafka) | confluent-kafka >= 1.8.2, <= 2.3.0 | No | experimental
+| [opentelemetry-instrumentation-confluent-kafka](./opentelemetry-instrumentation-confluent-kafka) | confluent-kafka >= 1.8.2, <= 2.4.0 | No | experimental
 | [opentelemetry-instrumentation-dbapi](./opentelemetry-instrumentation-dbapi) | dbapi | No | experimental
 | [opentelemetry-instrumentation-django](./opentelemetry-instrumentation-django) | django >= 1.10 | Yes | experimental
 | [opentelemetry-instrumentation-elasticsearch](./opentelemetry-instrumentation-elasticsearch) | elasticsearch >= 6.0 | No | experimental
diff --git a/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py b/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py
@@ -129,10 +129,10 @@ def client_response_hook(span: Span, scope: dict[str, Any], message: dict[str, A
 
 The name of the added span attribute will follow the format ``http.request.header.<header_name>`` where ``<header_name>``
 is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a
-single item list containing all the header values.
+list containing the header values.
 
 For example:
-``http.request.header.custom_request_header = ["<value1>,<value2>"]``
+``http.request.header.custom_request_header = ["<value1>", "<value2>"]``
 
 Response headers
 ****************
@@ -163,10 +163,10 @@ def client_response_hook(span: Span, scope: dict[str, Any], message: dict[str, A
 
 The name of the added span attribute will follow the format ``http.response.header.<header_name>`` where ``<header_name>``
 is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a
-single item list containing all the header values.
+list containing the header values.
 
 For example:
-``http.response.header.custom_response_header = ["<value1>,<value2>"]``
+``http.response.header.custom_response_header = ["<value1>", "<value2>"]``
 
 Sanitizing headers
 ******************
@@ -193,9 +193,10 @@ def client_response_hook(span: Span, scope: dict[str, Any], message: dict[str, A
 
 import typing
 import urllib
+from collections import defaultdict
 from functools import wraps
 from timeit import default_timer
-from typing import Any, Awaitable, Callable, Tuple
+from typing import Any, Awaitable, Callable, DefaultDict, Tuple
 
 from asgiref.compatibility import guarantee_single_callable
 
@@ -340,24 +341,19 @@ def collect_custom_headers_attributes(
     sanitize: SanitizeValue,
     header_regexes: list[str],
     normalize_names: Callable[[str], str],
-) -> dict[str, str]:
+) -> dict[str, list[str]]:
     """
     Returns custom HTTP request or response headers to be added into SERVER span as span attributes.
 
     Refer specifications:
      - https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-request-and-response-headers
     """
-    # Decode headers before processing.
-    headers: dict[str, str] = {}
+    headers: DefaultDict[str, list[str]] = defaultdict(list)
     raw_headers = scope_or_response_message.get("headers")
     if raw_headers:
-        for _key, _value in raw_headers:
-            key = _key.decode().lower()
-            value = _value.decode()
-            if key in headers:
-                headers[key] += f",{value}"
-            else:
-                headers[key] = value
+        for key, value in raw_headers:
+            # Decode headers before processing.
+            headers[key.decode()].append(value.decode())
 
     return sanitize.sanitize_header_values(
         headers,
@@ -707,7 +703,9 @@ async def otel_send(message: dict[str, Any]):
                 elif message["type"] == "websocket.send":
                     status_code = 200
                 if status_code:
-                    duration_attrs[SpanAttributes.HTTP_STATUS_CODE] = status_code
+                    duration_attrs[SpanAttributes.HTTP_STATUS_CODE] = (
+                        status_code
+                    )
 
                 if send_span.is_recording():
                     if message["type"] == "http.response.start":
diff --git a/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_custom_headers.py b/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_custom_headers.py
@@ -152,7 +152,8 @@ def test_http_repeat_request_headers_in_span_attributes(self):
         span_list = self.exporter.get_finished_spans()
         expected = {
             "http.request.header.custom_test_header_1": (
-                "test-header-value-1,test-header-value-2",
+                "test-header-value-1",
+                "test-header-value-2",
             ),
         }
         span = next(span for span in span_list if span.kind == SpanKind.SERVER)
@@ -225,7 +226,8 @@ def test_http_repeat_response_headers_in_span_attributes(self):
         span_list = self.exporter.get_finished_spans()
         expected = {
             "http.response.header.custom_test_header_1": (
-                "test-header-value-1,test-header-value-2",
+                "test-header-value-1",
+                "test-header-value-2",
             ),
         }
         span = next(span for span in span_list if span.kind == SpanKind.SERVER)
diff --git a/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py b/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py
@@ -327,7 +327,9 @@ def test_asgi_not_recording(self):
         mock_span = mock.Mock()
         mock_span.is_recording.return_value = False
         mock_tracer.start_as_current_span.return_value = mock_span
-        mock_tracer.start_as_current_span.return_value.__enter__ = mock.Mock(return_value=mock_span)
+        mock_tracer.start_as_current_span.return_value.__enter__ = mock.Mock(
+            return_value=mock_span
+        )
         mock_tracer.start_as_current_span.return_value.__exit__ = mock_span
         with mock.patch("opentelemetry.trace.get_tracer") as tracer:
             tracer.return_value = mock_tracer
@@ -804,7 +806,9 @@ def test_basic_metric_success_nonrecording_span(self):
         mock_span = mock.Mock()
         mock_span.is_recording.return_value = False
         mock_tracer.start_as_current_span.return_value = mock_span
-        mock_tracer.start_as_current_span.return_value.__enter__ = mock.Mock(return_value=mock_span)
+        mock_tracer.start_as_current_span.return_value.__enter__ = mock.Mock(
+            return_value=mock_span
+        )
         mock_tracer.start_as_current_span.return_value.__exit__ = mock_span
         with mock.patch("opentelemetry.trace.get_tracer") as tracer:
             tracer.return_value = mock_tracer
@@ -843,7 +847,9 @@ def test_basic_metric_success_nonrecording_span(self):
                                     self.assertAlmostEqual(
                                         duration, point.sum, delta=5
                                     )
-                                elif metric.name == "http.server.response.size":
+                                elif (
+                                    metric.name == "http.server.response.size"
+                                ):
                                     self.assertEqual(1024, point.sum)
                                 elif metric.name == "http.server.request.size":
                                     self.assertEqual(128, point.sum)
diff --git a/instrumentation/opentelemetry-instrumentation-confluent-kafka/pyproject.toml b/instrumentation/opentelemetry-instrumentation-confluent-kafka/pyproject.toml
@@ -32,7 +32,7 @@ dependencies = [
 
 [project.optional-dependencies]
 instruments = [
-  "confluent-kafka >= 1.8.2, <= 2.3.0",
+  "confluent-kafka >= 1.8.2, <= 2.4.0",
 ]
 
 [project.entry-points.opentelemetry_instrumentor]
diff --git a/instrumentation/opentelemetry-instrumentation-confluent-kafka/src/opentelemetry/instrumentation/confluent_kafka/package.py b/instrumentation/opentelemetry-instrumentation-confluent-kafka/src/opentelemetry/instrumentation/confluent_kafka/package.py
@@ -13,4 +13,4 @@
 # limitations under the License.
 
 
-_instruments = ("confluent-kafka >= 1.8.2, <= 2.3.0",)
+_instruments = ("confluent-kafka >= 1.8.2, <= 2.4.0",)
diff --git a/instrumentation/opentelemetry-instrumentation-confluent-kafka/test-requirements.txt b/instrumentation/opentelemetry-instrumentation-confluent-kafka/test-requirements.txt
@@ -1,5 +1,5 @@
 asgiref==3.7.2
-confluent-kafka==2.3.0
+confluent-kafka==2.4.0
 Deprecated==1.2.14
 importlib-metadata==6.11.0
 iniconfig==2.0.0
diff --git a/instrumentation/opentelemetry-instrumentation-fastapi/src/opentelemetry/instrumentation/fastapi/__init__.py b/instrumentation/opentelemetry-instrumentation-fastapi/src/opentelemetry/instrumentation/fastapi/__init__.py
@@ -115,7 +115,7 @@ def client_response_hook(span: Span, scope: dict[str, Any], message: dict[str, A
 single item list containing all the header values.
 
 For example:
-``http.request.header.custom_request_header = ["<value1>,<value2>"]``
+``http.request.header.custom_request_header = ["<value1>", "<value2>"]``
 
 Response headers
 ****************
@@ -146,10 +146,10 @@ def client_response_hook(span: Span, scope: dict[str, Any], message: dict[str, A
 
 The name of the added span attribute will follow the format ``http.response.header.<header_name>`` where ``<header_name>``
 is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a
-single item list containing all the header values.
+list containing the header values.
 
 For example:
-``http.response.header.custom_response_header = ["<value1>,<value2>"]``
+``http.response.header.custom_response_header = ["<value1>", "<value2>"]``
 
 Sanitizing headers
 ******************
diff --git a/instrumentation/opentelemetry-instrumentation-fastapi/tests/test_fastapi_instrumentation.py b/instrumentation/opentelemetry-instrumentation-fastapi/tests/test_fastapi_instrumentation.py
@@ -11,9 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import unittest
+from collections.abc import Mapping
 from timeit import default_timer
+from typing import Tuple
 from unittest.mock import patch
 
 import fastapi
@@ -557,6 +558,24 @@ def test_mark_span_internal_in_presence_of_span_from_other_framework(self):
         )
 
 
+class MultiMapping(Mapping):
+
+    def __init__(self, *items: Tuple[str, str]):
+        self._items = items
+
+    def __len__(self):
+        return len(self._items)
+
+    def __getitem__(self, __key):
+        raise NotImplementedError("use .items() instead")
+
+    def __iter__(self):
+        raise NotImplementedError("use .items() instead")
+
+    def items(self):
+        return self._items
+
+
 @patch.dict(
     "os.environ",
     {
@@ -583,13 +602,15 @@ def _create_app():
 
         @app.get("/foobar")
         async def _():
-            headers = {
-                "custom-test-header-1": "test-header-value-1",
-                "custom-test-header-2": "test-header-value-2",
-                "my-custom-regex-header-1": "my-custom-regex-value-1,my-custom-regex-value-2",
-                "My-Custom-Regex-Header-2": "my-custom-regex-value-3,my-custom-regex-value-4",
-                "My-Secret-Header": "My Secret Value",
-            }
+            headers = MultiMapping(
+                ("custom-test-header-1", "test-header-value-1"),
+                ("custom-test-header-2", "test-header-value-2"),
+                ("my-custom-regex-header-1", "my-custom-regex-value-1"),
+                ("my-custom-regex-header-1", "my-custom-regex-value-2"),
+                ("My-Custom-Regex-Header-2", "my-custom-regex-value-3"),
+                ("My-Custom-Regex-Header-2", "my-custom-regex-value-4"),
+                ("My-Secret-Header", "My Secret Value"),
+            )
             content = {"message": "hello world"}
             return JSONResponse(content=content, headers=headers)
 
@@ -665,10 +686,12 @@ def test_http_custom_response_headers_in_span_attributes(self):
                 "test-header-value-2",
             ),
             "http.response.header.my_custom_regex_header_1": (
-                "my-custom-regex-value-1,my-custom-regex-value-2",
+                "my-custom-regex-value-1",
+                "my-custom-regex-value-2",
             ),
             "http.response.header.my_custom_regex_header_2": (
-                "my-custom-regex-value-3,my-custom-regex-value-4",
+                "my-custom-regex-value-3",
+                "my-custom-regex-value-4",
             ),
             "http.response.header.my_secret_header": ("[REDACTED]",),
         }
diff --git a/instrumentation/opentelemetry-instrumentation-pymysql/tests/test_pymysql_integration.py b/instrumentation/opentelemetry-instrumentation-pymysql/tests/test_pymysql_integration.py
@@ -17,6 +17,7 @@
 import pymysql
 
 import opentelemetry.instrumentation.pymysql
+from opentelemetry import trace as trace_api
 from opentelemetry.instrumentation.pymysql import PyMySQLInstrumentor
 from opentelemetry.sdk import resources
 from opentelemetry.test.test_base import TestBase
@@ -78,6 +79,20 @@ def test_custom_tracer_provider(self, mock_connect):
 
         self.assertIs(span.resource, resource)
 
+    @mock.patch("pymysql.connect")
+    # pylint: disable=unused-argument
+    def test_no_op_tracer_provider(self, mock_connect):
+        PyMySQLInstrumentor().instrument(
+            tracer_provider=trace_api.NoOpTracerProvider()
+        )
+        cnx = pymysql.connect(database="test")
+        cursor = cnx.cursor()
+        query = "SELECT * FROM test"
+        cursor.execute(query)
+
+        spans_list = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(spans_list), 0)
+
     @mock.patch("pymysql.connect")
     # pylint: disable=unused-argument
     def test_instrument_connection(self, mock_connect):
diff --git a/instrumentation/opentelemetry-instrumentation-starlette/src/opentelemetry/instrumentation/starlette/__init__.py b/instrumentation/opentelemetry-instrumentation-starlette/src/opentelemetry/instrumentation/starlette/__init__.py
@@ -110,10 +110,10 @@ def client_response_hook(span: Span, scope: dict[str, Any], message: dict[str, A
 
 The name of the added span attribute will follow the format ``http.request.header.<header_name>`` where ``<header_name>``
 is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a
-single item list containing all the header values.
+list containing the header values.
 
 For example:
-``http.request.header.custom_request_header = ["<value1>,<value2>"]``
+``http.request.header.custom_request_header = ["<value1>", "<value2>"]``
 
 Response headers
 ****************
@@ -144,10 +144,10 @@ def client_response_hook(span: Span, scope: dict[str, Any], message: dict[str, A
 
 The name of the added span attribute will follow the format ``http.response.header.<header_name>`` where ``<header_name>``
 is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a
-single item list containing all the header values.
+list containing the header values.
 
 For example:
-``http.response.header.custom_response_header = ["<value1>,<value2>"]``
+``http.response.header.custom_response_header = ["<value1>", "<value2>"]``
 
 Sanitizing headers
 ******************
diff --git a/instrumentation/opentelemetry-instrumentation-system-metrics/pyproject.toml b/instrumentation/opentelemetry-instrumentation-system-metrics/pyproject.toml
@@ -27,7 +27,7 @@ classifiers = [
 dependencies = [
   "opentelemetry-instrumentation == 0.47b0.dev",
   "opentelemetry-api ~= 1.11",
-  "psutil ~= 5.9",
+  "psutil >= 5.9.0, < 7",
 ]
 
 [project.optional-dependencies]
diff --git a/instrumentation/opentelemetry-instrumentation-system-metrics/test-requirements.txt b/instrumentation/opentelemetry-instrumentation-system-metrics/test-requirements.txt
@@ -4,7 +4,7 @@ importlib-metadata==6.11.0
 iniconfig==2.0.0
 packaging==24.0
 pluggy==1.5.0
-psutil==5.9.8
+psutil==6.0.0
 py-cpuinfo==9.0.0
 pytest==7.4.4
 pytest-benchmark==4.0.0
diff --git a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py
@@ -65,7 +65,7 @@
         "instrumentation": "opentelemetry-instrumentation-celery==0.47b0.dev",
     },
     {
-        "library": "confluent-kafka >= 1.8.2, <= 2.3.0",
+        "library": "confluent-kafka >= 1.8.2, <= 2.4.0",
         "instrumentation": "opentelemetry-instrumentation-confluent-kafka==0.47b0.dev",
     },
     {
diff --git a/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py b/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py

Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,7 @@ dependencies = [`
`32`	`32`
`33`	`33`	`[project.optional-dependencies]`
`34`	`34`	`instruments = [`
`35`		`- "confluent-kafka >= 1.8.2, <= 2.3.0",`
	`35`	`+ "confluent-kafka >= 1.8.2, <= 2.4.0",`
`36`	`36`	`]`
`37`	`37`
`38`	`38`	`[project.entry-points.opentelemetry_instrumentor]`
Original file line number	Diff line number	Diff line change
`@@ -13,4 +13,4 @@`
`13`	`13`	`# limitations under the License.`
`14`	`14`
`15`	`15`
`16`		`-_instruments = ("confluent-kafka >= 1.8.2, <= 2.3.0",)`
	`16`	`+_instruments = ("confluent-kafka >= 1.8.2, <= 2.4.0",)`