Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/system-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ jobs:
persist-credentials: false
repository: 'DataDog/system-tests'
# Automatically managed, use scripts/update-system-tests-version to update
ref: '4bece37901d6241526eb788bdac6f2887251cdcd'
ref: 'da285213e99805433b99f8ea9d1e57f68e2b8e3d'

- name: Download wheels to binaries directory
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
Expand Down Expand Up @@ -94,7 +94,7 @@ jobs:
persist-credentials: false
repository: 'DataDog/system-tests'
# Automatically managed, use scripts/update-system-tests-version to update
ref: '4bece37901d6241526eb788bdac6f2887251cdcd'
ref: 'da285213e99805433b99f8ea9d1e57f68e2b8e3d'

- name: Build runner
uses: ./.github/actions/install_runner
Expand Down Expand Up @@ -283,7 +283,7 @@ jobs:
persist-credentials: false
repository: 'DataDog/system-tests'
# Automatically managed, use scripts/update-system-tests-version to update
ref: '4bece37901d6241526eb788bdac6f2887251cdcd'
ref: 'da285213e99805433b99f8ea9d1e57f68e2b8e3d'
- name: Download wheels to binaries directory
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
Expand Down
2 changes: 1 addition & 1 deletion .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ variables:
DD_VPA_TEMPLATE: "vpa-template-cpu-p70-10percent-2x-oom-min-cap"
# CI_DEBUG_SERVICES: "true"
# Automatically managed, use scripts/update-system-tests-version to update
SYSTEM_TESTS_REF: "4bece37901d6241526eb788bdac6f2887251cdcd"
SYSTEM_TESTS_REF: "da285213e99805433b99f8ea9d1e57f68e2b8e3d"

default:
interruptible: true
Expand Down
1 change: 1 addition & 0 deletions benchmarks/bm/flask_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ def setup(self):
"DD_APPSEC_ENABLED": str(self.appsec_enabled),
"DD_IAST_ENABLED": str(self.iast_enabled),
"DD_TELEMETRY_METRICS_ENABLED": str(self.telemetry_metrics_enabled),
"DD_TRACE_RESOURCE_RENAMING_ENABLED": str(self.resource_renaming_enabled),
}
)

Expand Down
4 changes: 4 additions & 0 deletions benchmarks/django_simple/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ baseline: &baseline
django_instrument_templates: true
native_writer: false
django_minimal: false
resource_renaming_enabled: false
tracer: &tracer
<<: *baseline
tracer_enabled: true
Expand Down Expand Up @@ -56,3 +57,6 @@ tracer-native:
tracer-minimal:
<<: *tracer
django_minimal: true
resource-renaming:
<<: *tracer
resource_renaming_enabled: true
3 changes: 3 additions & 0 deletions benchmarks/django_simple/scenario.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ class DjangoSimple(bm.Scenario):
django_instrument_templates: bool
native_writer: bool
django_minimal: bool
resource_renaming_enabled: bool

def run(self):
os.environ["DJANGO_SETTINGS_MODULE"] = "app"
Expand Down Expand Up @@ -49,6 +50,8 @@ def run(self):
os.environ.update({"DD_EXCEPTION_REPLAY_ENABLED": "1"})
if self.native_writer:
os.environ.update({"_DD_TRACE_WRITER_NATIVE": "1"})
if self.resource_renaming_enabled:
os.environ.update({"DD_TRACE_RESOURCE_RENAMING_ENABLED": "1"})

# This will not work with gevent workers as the gevent hub has not been
# initialized when this hook is called.
Expand Down
1 change: 1 addition & 0 deletions benchmarks/errortracking_flask_sqli/scenario.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ class ErrorTrackingFlaskSQLi(bm.Scenario, FlaskScenarioMixin):
telemetry_metrics_enabled: bool
errortracking_enabled: str
native_writer: bool
resource_renaming_enabled: bool

def run(self):
app = self.create_app()
Expand Down
6 changes: 5 additions & 1 deletion benchmarks/flask_simple/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ baseline: &baseline
post_request: false
telemetry_metrics_enabled: false
native_writer: false
resource_renaming_enabled: false
tracer: &tracer
<<: *baseline
tracer_enabled: true
Expand All @@ -32,7 +33,10 @@ appsec-telemetry:
tracer-native:
<<: *tracer
native_writer: true
# The scenarios below produce inconsistent results. We plan to
resource-renaming:
<<: *tracer
resource_renaming_enabled: true
# The scenarios below produce inconsistent results. We plan to
# disbable these scenarios until the root cause is identified:
# tracer-and-profiler:
# <<: *baseline
Expand Down
1 change: 1 addition & 0 deletions benchmarks/flask_simple/scenario.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ class FlaskSimple(bm.Scenario, FlaskScenarioMixin):
telemetry_metrics_enabled: bool
errortracking_enabled: str
native_writer: bool
resource_renaming_enabled: bool

def run(self):
app = self.create_app()
Expand Down
1 change: 1 addition & 0 deletions benchmarks/flask_sqli/scenario.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ class FlaskSQLi(bm.Scenario, FlaskScenarioMixin):
telemetry_metrics_enabled: bool
errortracking_enabled: str
native_writer: bool
resource_renaming_enabled: bool

def run(self):
app = self.create_app()
Expand Down
77 changes: 77 additions & 0 deletions ddtrace/_trace/processor/resource_renaming.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import re
from typing import List
from typing import Optional
from urllib.parse import urlparse

from ddtrace._trace.processor import SpanProcessor
from ddtrace.ext import SpanTypes
from ddtrace.ext import http
from ddtrace.internal.logger import get_logger
from ddtrace.settings._config import config


log = get_logger(__name__)


class ResourceRenamingProcessor(SpanProcessor):
def __init__(self):
self._INT_RE = re.compile(r"^[1-9][0-9]+$")
self._INT_ID_RE = re.compile(r"^(?=.*[0-9].*)[0-9._-]{3,}$")
self._HEX_RE = re.compile(r"^(?=.*[0-9].*)[A-Fa-f0-9]{6,}$")
self._HEX_ID_RE = re.compile(r"^(?=.*[0-9].*)[A-Fa-f0-9._-]{6,}$")
self._STR_RE = re.compile(r"^(.{20,}|.*[%&'()*+,:=@].*)$")

def _compute_simplified_endpoint_path_element(self, elem: str) -> str:
"""Applies the parameter replacement rules to a single path element."""
if self._INT_RE.fullmatch(elem):
return "{param:int}"
if self._INT_ID_RE.fullmatch(elem):
return "{param:int_id}"
if self._HEX_RE.fullmatch(elem):
return "{param:hex}"
if self._HEX_ID_RE.fullmatch(elem):
return "{param:hex_id}"
if self._STR_RE.fullmatch(elem):
return "{param:str}"
return elem

def _compute_simplified_endpoint(self, url: Optional[str]) -> str:
"""Extracts and simplifies the path from an HTTP URL."""
if not url:
return "/"

try:
parsed_url = urlparse(url)
except ValueError as e:
log.error("Failed to parse http.url tag when processing span for resource renaming: %s", e)
return "/"
path = parsed_url.path
if not path or path == "/":
return "/"

elements: List[str] = []
for part in path.split("/"):
if part:
elements.append(part)
if len(elements) >= 8:
break

if not elements:
return "/"

elements = [self._compute_simplified_endpoint_path_element(elem) for elem in elements]
return "/" + "/".join(elements)

def on_span_start(self, span):
pass

def on_span_finish(self, span):
if not span._is_top_level or span.span_type not in (SpanTypes.WEB, SpanTypes.HTTP, SpanTypes.SERVERLESS):
return

route = span.get_tag(http.ROUTE)

if not route or config._trace_resource_renaming_always_simplified_endpoint:
url = span.get_tag(http.URL)
endpoint = self._compute_simplified_endpoint(url)
span.set_tag_str(http.ENDPOINT, endpoint)
4 changes: 4 additions & 0 deletions ddtrace/_trace/tracer.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from ddtrace._trace.processor import SpanProcessor
from ddtrace._trace.processor import TopLevelSpanProcessor
from ddtrace._trace.processor import TraceProcessor
from ddtrace._trace.processor.resource_renaming import ResourceRenamingProcessor
from ddtrace._trace.provider import BaseContextProvider
from ddtrace._trace.provider import DefaultContextProvider
from ddtrace._trace.span import Span
Expand Down Expand Up @@ -74,6 +75,9 @@ def _default_span_processors_factory(
span_processors: List[SpanProcessor] = []
span_processors += [TopLevelSpanProcessor()]

if config._trace_resource_renaming_enabled:
span_processors.append(ResourceRenamingProcessor())

# When using the NativeWriter stats are computed by the native code.
if config._trace_compute_stats and not config._trace_writer_native:
# Inline the import to avoid pulling in ddsketch or protobuf
Expand Down
1 change: 1 addition & 0 deletions ddtrace/ext/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
CLIENT_IP = "http.client_ip"
ROUTE = "http.route"
REFERRER_HOSTNAME = "http.referrer_hostname"
ENDPOINT = "http.endpoint"

# HTTP headers
REFERER_HEADER = "referer"
Expand Down
10 changes: 8 additions & 2 deletions ddtrace/internal/processor/stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ def _is_measured(span: Span) -> bool:
str, # type
int, # http status code
bool, # synthetics request
str, # http method
str, # http endpoint
]


Expand All @@ -73,8 +75,10 @@ def _span_aggr_key(span: Span) -> SpanAggrKey:
resource = span.resource or ""
_type = span.span_type or ""
status_code = span.get_tag("http.status_code") or 0
method = span.get_tag("http.method") or ""
endpoint = span.get_tag("http.endpoint") or span.get_tag("http.route") or ""
synthetics = span.context.dd_origin == "synthetics"
return span.name, service, resource, _type, int(status_code), synthetics
return (span.name, service, resource, _type, int(status_code), synthetics, method, endpoint)


class SpanStatsProcessorV06(PeriodicService, SpanProcessor):
Expand Down Expand Up @@ -157,12 +161,14 @@ def _serialize_buckets(self) -> List[Dict]:
serialized_bucket_keys.append(bucket_time_ns)

for aggr_key, stat_aggr in bucket.items():
name, service, resource, _type, http_status, synthetics = aggr_key
name, service, resource, _type, http_status, synthetics, http_method, http_endpoint = aggr_key
serialized_bucket = {
"Name": compat.ensure_text(name),
"Resource": compat.ensure_text(resource),
"Synthetics": synthetics,
"HTTPStatusCode": http_status,
"HTTPMethod": http_method,
"HTTPEndpoint": http_endpoint,
"Hits": stat_aggr.hits,
"TopLevelHits": stat_aggr.top_level_hits,
"Duration": stat_aggr.duration,
Expand Down
8 changes: 8 additions & 0 deletions ddtrace/settings/_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -661,6 +661,14 @@ def __init__(self):
self._inferred_proxy_services_enabled = _get_config("DD_TRACE_INFERRED_PROXY_SERVICES_ENABLED", False, asbool)
self._trace_safe_instrumentation_enabled = _get_config("DD_TRACE_SAFE_INSTRUMENTATION_ENABLED", False, asbool)

# Resource renaming
self._trace_resource_renaming_enabled = _get_config(
"DD_TRACE_RESOURCE_RENAMING_ENABLED", default=False, modifier=asbool
)
self._trace_resource_renaming_always_simplified_endpoint = _get_config(
"DD_TRACE_RESOURCE_RENAMING_ALWAYS_SIMPLIFIED_ENDPOINT", default=False, modifier=asbool
)

def __getattr__(self, name) -> Any:
if name in self._config:
return self._config[name].value()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
---
features:
- |
tracing: Added support for resource renaming, a experimental feature that enables automatic renaming of the resource field of spans for web requests from the Datadog user interface when the resource name does not correctly represent an endpoint.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we should put the env var here to show how to enable.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it doesn't seem like this feature actually renames the resource, right? it looks like it just optionally adds an additional tag http.endpoint?

maybe I don't understand that tag very well

2 changes: 2 additions & 0 deletions tests/telemetry/test_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,8 @@ def test_app_started_event_configuration_override(test_agent_session, run_python
{"name": "DD_TRACE_PROPAGATION_STYLE_INJECT", "origin": "env_var", "value": "tracecontext"},
{"name": "DD_TRACE_RATE_LIMIT", "origin": "env_var", "value": 50},
{"name": "DD_TRACE_REPORT_HOSTNAME", "origin": "default", "value": False},
{"name": "DD_TRACE_RESOURCE_RENAMING_ALWAYS_SIMPLIFIED_ENDPOINT", "origin": "default", "value": False},
{"name": "DD_TRACE_RESOURCE_RENAMING_ENABLED", "origin": "default", "value": False},
{"name": "DD_TRACE_SAFE_INSTRUMENTATION_ENABLED", "origin": "default", "value": False},
{
"name": "DD_TRACE_SAMPLING_RULES",
Expand Down
Loading
Loading