fix(asm): decouple appsec (#12212)

Second part of #12198

- Ensure we don't load appsec modules if appsec is disabled or
unavailable (except for a few safe modules).
- Ensure we don't load iast modules if iast is disabled or unavailable.
- Ensure all initialisation logic for enable flags are handled in
ddtrace.settings.asm (small factorisation)
- Add test on module loading with a mini flask application in
`tests/appsec/integrations/flask_tests/test_appsec_loading_modules.py`,
checking with all possible combinations of appsec/iast/aws_lambda
enabled or disabled.
- remove dead code in `ddtrace/contrib/internal/langchain/patch.py`

APPSEC-56626

(should be backported to 3.0 when possible)

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Author: christophe-papazian

ddtrace/appsec/__init__.py

@@ -1,5 +1,6 @@
+# this module must not load any other unsafe appsec module directly
+
 from ddtrace.internal import core
-from ddtrace.settings.asm import config as asm_config
 
 
 _APPSEC_TO_BE_LOADED = True
@@ -28,7 +29,9 @@ def load_iast():
 
 def load_common_appsec_modules():
     """Lazily load the common module patches."""
-    if (asm_config._ep_enabled and asm_config._asm_enabled) or asm_config._iast_enabled:
+    from ddtrace.settings.asm import config as asm_config
+
+    if asm_config._load_modules:
         from ddtrace.appsec._common_module_patches import patch_common_modules
 
         patch_common_modules()

ddtrace/appsec/_asm_request_context.py

@@ -15,9 +15,6 @@
 from ddtrace.appsec._constants import APPSEC
 from ddtrace.appsec._constants import EXPLOIT_PREVENTION
 from ddtrace.appsec._constants import SPAN_DATA_NAMES
-from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled
-from ddtrace.appsec._iast._taint_tracking import OriginType
-from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject
 from ddtrace.appsec._utils import add_context_log
 from ddtrace.appsec._utils import get_triggers
 from ddtrace.internal import core
@@ -28,6 +25,16 @@
 from ddtrace.trace import Span
 
 
+if asm_config._iast_enabled:
+    from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled
+    from ddtrace.appsec._iast._taint_tracking import OriginType
+    from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject
+else:
+
+    def is_iast_request_enabled() -> bool:
+        return False
+
+
 if TYPE_CHECKING:
     from ddtrace.appsec._ddwaf import DDWaf_info
     from ddtrace.appsec._ddwaf import DDWaf_result

ddtrace/appsec/_common_module_patches.py

@@ -44,6 +44,10 @@ def is_iast_request_enabled() -> bool:
 
 def patch_common_modules():
     global _is_patched
+    # ensure that the subprocess patch is applied even after one click activation
+    subprocess_patch.patch()
+    subprocess_patch.add_str_callback(_RASP_SYSTEM, wrapped_system_5542593D237084A7)
+    subprocess_patch.add_lst_callback(_RASP_POPEN, popen_FD233052260D8B4D)
     if _is_patched:
         return
     # for testing purposes, we need to update is_iast_request_enabled
@@ -60,10 +64,6 @@ def is_iast_request_enabled() -> bool:
     try_wrap_function_wrapper("urllib.request", "OpenerDirector.open", wrapped_open_ED4CF71136E15EBF)
     try_wrap_function_wrapper("_io", "BytesIO.read", wrapped_read_F3E51D71B4EC16EF)
     try_wrap_function_wrapper("_io", "StringIO.read", wrapped_read_F3E51D71B4EC16EF)
-    # ensure that the subprocess patch is applied even after one click activation
-    subprocess_patch.patch()
-    subprocess_patch.add_str_callback(_RASP_SYSTEM, wrapped_system_5542593D237084A7)
-    subprocess_patch.add_lst_callback(_RASP_POPEN, popen_FD233052260D8B4D)
     core.on("asm.block.dbapi.execute", execute_4C9BAC8E228EB347)
     if asm_config._iast_enabled:
         from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink

ddtrace/appsec/_constants.py

@@ -1,3 +1,5 @@
+# this module must not load any other unsafe appsec module directly
+
 import os
 from re import Match
 import sys

ddtrace/appsec/_utils.py

@@ -1,10 +1,13 @@
+# this module must not load any other unsafe appsec module directly
+
 import logging
 import sys
 from typing import Any
 import uuid
 
 from ddtrace.appsec._constants import API_SECURITY
 from ddtrace.appsec._constants import APPSEC
+from ddtrace.appsec._constants import SPAN_DATA_NAMES
 from ddtrace.internal._unpatched import unpatched_json_loads
 from ddtrace.internal.compat import to_unicode
 from ddtrace.internal.logger import get_logger
@@ -21,7 +24,6 @@ def parse_response_body(raw_body):
     import xmltodict
 
     from ddtrace.appsec import _asm_request_context
-    from ddtrace.appsec._constants import SPAN_DATA_NAMES
     from ddtrace.contrib.internal.trace_utils import _get_header_value_case_insensitive
 
     if not raw_body:

ddtrace/contrib/internal/httplib/patch.py

@@ -5,7 +5,6 @@
 import wrapt
 
 from ddtrace import config
-from ddtrace.appsec._common_module_patches import wrapped_request_D8CB81E472AF98A2 as _wrap_request_asm
 from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY
 from ddtrace.constants import SPAN_KIND
 from ddtrace.contrib import trace_utils
@@ -77,12 +76,14 @@ def _wrap_getresponse(func, instance, args, kwargs):
 
 
 def _call_asm_wrap(func, instance, *args, **kwargs):
+    from ddtrace.appsec._common_module_patches import wrapped_request_D8CB81E472AF98A2 as _wrap_request_asm
+
     _wrap_request_asm(func, instance, args, kwargs)
 
 
 def _wrap_request(func, instance, args, kwargs):
     # Use any attached tracer if available, otherwise use the global tracer
-    if asm_config._iast_enabled or asm_config._asm_enabled:
+    if asm_config._iast_enabled or (asm_config._asm_enabled and asm_config._ep_enabled):
         func_to_call = functools.partial(_call_asm_wrap, func, instance)
     else:
         func_to_call = func

ddtrace/contrib/internal/langchain/patch.py

@@ -1090,29 +1090,6 @@ def unpatch():
     delattr(langchain, "_datadog_integration")
 
 
-def taint_outputs(instance, inputs, outputs):
-    from ddtrace.appsec._iast._metrics import _set_iast_error_metric
-    from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges
-    from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject
-
-    try:
-        ranges = None
-        for key in filter(lambda x: x in inputs, instance.input_keys):
-            input_val = inputs.get(key)
-            if input_val:
-                ranges = get_tainted_ranges(input_val)
-                if ranges:
-                    break
-
-        if ranges:
-            source = ranges[0].source
-            for key in filter(lambda x: x in outputs, instance.output_keys):
-                output_value = outputs[key]
-                outputs[key] = taint_pyobject(output_value, source.name, source.value, source.origin)
-    except Exception as e:
-        _set_iast_error_metric("IAST propagation error. langchain taint_outputs. {}".format(e))
-
-
 def taint_parser_output(func, instance, args, kwargs):
     from ddtrace.appsec._iast._metrics import _set_iast_error_metric
     from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges

ddtrace/contrib/internal/mysql/patch.py

@@ -4,8 +4,6 @@
 import wrapt
 
 from ddtrace import config
-from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink
-from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION
 from ddtrace.contrib.dbapi import TracedConnection
 from ddtrace.contrib.internal.trace_utils import _convert_to_string
 from ddtrace.ext import db
@@ -51,6 +49,9 @@ def patch():
         mysql.connector.Connect = mysql.connector.connect
 
     if asm_config._iast_enabled:
+        from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink
+        from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION
+
         _set_metric_iast_instrumented_sink(VULN_SQL_INJECTION)
     mysql.connector._datadog_patch = True
 

ddtrace/contrib/internal/mysqldb/patch.py

@@ -4,8 +4,6 @@
 from wrapt import wrap_function_wrapper as _w
 
 from ddtrace import config
-from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink
-from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION
 from ddtrace.constants import _SPAN_MEASURED_KEY
 from ddtrace.constants import SPAN_KIND
 from ddtrace.contrib.dbapi import TracedConnection
@@ -67,6 +65,9 @@ def patch():
         _w("MySQLdb", "connect", _connect)
 
     if asm_config._iast_enabled:
+        from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink
+        from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION
+
         _set_metric_iast_instrumented_sink(VULN_SQL_INJECTION)
 
 

ddtrace/contrib/internal/pytest/_plugin_v2.py

@@ -60,6 +60,7 @@
 from ddtrace.internal.test_visibility.api import InternalTestSession
 from ddtrace.internal.test_visibility.api import InternalTestSuite
 from ddtrace.internal.test_visibility.coverage_lines import CoverageLines
+from ddtrace.settings.asm import config as asm_config
 from ddtrace.vendor.debtcollector import deprecate
 
 
@@ -574,9 +575,10 @@ def _pytest_terminal_summary_post_yield(terminalreporter, failed_reports_initial
 def pytest_terminal_summary(terminalreporter, exitstatus, config):
     """Report flaky or failed tests"""
     try:
-        from ddtrace.appsec._iast._pytest_plugin import print_iast_report
+        if asm_config._iast_enabled:
+            from ddtrace.appsec._iast._pytest_plugin import print_iast_report
 
-        print_iast_report(terminalreporter)
+            print_iast_report(terminalreporter)
     except Exception:  # noqa: E722
         log.debug("Encountered error during code security summary", exc_info=True)
 

ddtrace/contrib/internal/pytest/plugin.py

@@ -17,13 +17,16 @@
 import pytest
 
 from ddtrace import config
-from ddtrace.appsec._iast._pytest_plugin import ddtrace_iast  # noqa:F401
 from ddtrace.contrib.internal.pytest._utils import _USE_PLUGIN_V2
 from ddtrace.contrib.internal.pytest._utils import _extract_span
 from ddtrace.contrib.internal.pytest._utils import _pytest_version_supports_itr
 from ddtrace.settings.asm import config as asm_config
 
 
+if asm_config._iast_enabled:
+    from ddtrace.appsec._iast._pytest_plugin import ddtrace_iast  # noqa:F401
+
+
 # pytest default settings
 config._add(
     "pytest",

ddtrace/contrib/internal/requests/patch.py

@@ -4,9 +4,6 @@
 from wrapt import wrap_function_wrapper as _w
 
 from ddtrace import config
-from ddtrace.appsec._common_module_patches import wrapped_request_D8CB81E472AF98A2 as _wrap_request
-from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink
-from ddtrace.appsec._iast.constants import VULN_SSRF
 from ddtrace.contrib.internal.trace_utils import unwrap as _u
 from ddtrace.internal.schema import schematize_service_name
 from ddtrace.internal.utils.formats import asbool
@@ -46,10 +43,16 @@ def patch():
 
     _w("requests", "Session.send", _wrap_send)
     # IAST needs to wrap this function because `Session.send` is too late
-    _w("requests", "Session.request", _wrap_request)
+    if asm_config._load_modules:
+        from ddtrace.appsec._common_module_patches import wrapped_request_D8CB81E472AF98A2 as _wrap_request
+
+        _w("requests", "Session.request", _wrap_request)
     Pin(_config=config.requests).onto(requests.Session)
 
     if asm_config._iast_enabled:
+        from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink
+        from ddtrace.appsec._iast.constants import VULN_SSRF
+
         _set_metric_iast_instrumented_sink(VULN_SSRF)
 
 
@@ -59,5 +62,13 @@ def unpatch():
         return
     requests.__datadog_patch = False
 
-    _u(requests.Session, "request")
-    _u(requests.Session, "send")
+    try:
+        _u(requests.Session, "request")
+    except AttributeError:
+        # It was not patched
+        pass
+    try:
+        _u(requests.Session, "send")
+    except AttributeError:
+        # It was not patched
+        pass

ddtrace/contrib/internal/sqlalchemy/patch.py

@@ -1,8 +1,6 @@
 import sqlalchemy
 from wrapt import wrap_function_wrapper as _w
 
-from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink
-from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION
 from ddtrace.contrib.internal.trace_utils import unwrap
 from ddtrace.settings.asm import config as asm_config
 
@@ -24,6 +22,9 @@ def patch():
     _w("sqlalchemy.engine", "create_engine", _wrap_create_engine)
 
     if asm_config._iast_enabled:
+        from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink
+        from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION
+
         _set_metric_iast_instrumented_sink(VULN_SQL_INJECTION)
 
 

ddtrace/contrib/internal/sqlite3/patch.py

@@ -5,8 +5,6 @@
 import wrapt
 
 from ddtrace import config
-from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink
-from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION
 from ddtrace.contrib.dbapi import FetchTracedCursor
 from ddtrace.contrib.dbapi import TracedConnection
 from ddtrace.contrib.dbapi import TracedCursor
@@ -47,6 +45,9 @@ def patch():
     sqlite3.dbapi2.connect = wrapped
 
     if asm_config._iast_enabled:
+        from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink
+        from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION
+
         _set_metric_iast_instrumented_sink(VULN_SQL_INJECTION)
 
 

ddtrace/contrib/internal/subprocess/patch.py

@@ -58,15 +58,15 @@ def del_lst_callback(name: str):
 
 
 def patch() -> List[str]:
-    if not (asm_config._asm_enabled or asm_config._iast_enabled):
+    if not asm_config._load_modules:
         return []
     patched: List[str] = []
 
     import os  # nosec
     import subprocess  # nosec
 
     should_patch_system = not trace_utils.iswrapped(os.system)
-    should_patch_fork = not trace_utils.iswrapped(os.fork)
+    should_patch_fork = (not trace_utils.iswrapped(os.fork)) if hasattr(os, "fork") else False
     spawnvef = getattr(os, "_spawnvef", None)
     should_patch_spawnvef = spawnvef is not None and not trace_utils.iswrapped(spawnvef)
 
@@ -316,18 +316,19 @@ def unpatch() -> None:
     import os  # nosec
     import subprocess  # nosec
 
-    trace_utils.unwrap(os, "system")
-    trace_utils.unwrap(os, "_spawnvef")
-    trace_utils.unwrap(subprocess.Popen, "__init__")
-    trace_utils.unwrap(subprocess.Popen, "wait")
+    for obj, attr in [(os, "system"), (os, "_spawnvef"), (subprocess.Popen, "__init__"), (subprocess.Popen, "wait")]:
+        try:
+            trace_utils.unwrap(obj, attr)
+        except AttributeError:
+            pass
 
     SubprocessCmdLine._clear_cache()
 
 
 @trace_utils.with_traced_module
 def _traced_ossystem(module, pin, wrapped, instance, args, kwargs):
     try:
-        if asm_config._bypass_instrumentation_for_waf:
+        if asm_config._bypass_instrumentation_for_waf or not (asm_config._asm_enabled or asm_config._iast_enabled):
             return wrapped(*args, **kwargs)
         if isinstance(args[0], str):
             for callback in _STR_CALLBACKS.values():
@@ -351,6 +352,8 @@ def _traced_ossystem(module, pin, wrapped, instance, args, kwargs):
 
 @trace_utils.with_traced_module
 def _traced_fork(module, pin, wrapped, instance, args, kwargs):
+    if not (asm_config._asm_enabled or asm_config._iast_enabled):
+        return wrapped(*args, **kwargs)
     try:
         with pin.tracer.trace(COMMANDS.SPAN_NAME, resource="fork", span_type=SpanTypes.SYSTEM) as span:
             span.set_tag(COMMANDS.EXEC, ["os.fork"])
@@ -366,6 +369,8 @@ def _traced_fork(module, pin, wrapped, instance, args, kwargs):
 
 @trace_utils.with_traced_module
 def _traced_osspawn(module, pin, wrapped, instance, args, kwargs):
+    if not (asm_config._asm_enabled or asm_config._iast_enabled):
+        return wrapped(*args, **kwargs)
     try:
         mode, file, func_args, _, _ = args
         if isinstance(func_args, (list, tuple, str)):
@@ -395,7 +400,7 @@ def _traced_osspawn(module, pin, wrapped, instance, args, kwargs):
 @trace_utils.with_traced_module
 def _traced_subprocess_init(module, pin, wrapped, instance, args, kwargs):
     try:
-        if asm_config._bypass_instrumentation_for_waf:
+        if asm_config._bypass_instrumentation_for_waf or not (asm_config._asm_enabled or asm_config._iast_enabled):
             return wrapped(*args, **kwargs)
         cmd_args = args[0] if len(args) else kwargs["args"]
         if isinstance(cmd_args, (list, tuple, str)):
@@ -429,7 +434,7 @@ def _traced_subprocess_init(module, pin, wrapped, instance, args, kwargs):
 @trace_utils.with_traced_module
 def _traced_subprocess_wait(module, pin, wrapped, instance, args, kwargs):
     try:
-        if asm_config._bypass_instrumentation_for_waf:
+        if asm_config._bypass_instrumentation_for_waf or not (asm_config._asm_enabled or asm_config._iast_enabled):
             return wrapped(*args, **kwargs)
         binary = core.get_item("subprocess_popen_binary")