Skip to content

feat(profiling): add serverless scheduler and conditionally use it #3870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 39 commits into from
Aug 8, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
90df359
feat: WIP profiling support for Python
astuyve May 23, 2022
0f3fd52
wip: function name as tag
astuyve May 31, 2022
c06cd4f
feat: custom serverless scheduler to wake up every second and check i…
astuyve Jun 1, 2022
5c9ec7b
feat: Conditionally use serverless scheduler. Refactor to inherit fro…
astuyve Jun 10, 2022
cf15cd0
fix: use 60s, not 61s
astuyve Jun 10, 2022
6191541
feat: test. Back to 60s
astuyve Jun 10, 2022
80c7133
feat: Fix double flushing issue by comparing last export time as well…
astuyve Jun 23, 2022
9329c26
feat: riot formatter
astuyve Jun 23, 2022
5ecd1a2
fix: _interval should be int, not float
astuyve Jun 23, 2022
d104d70
feat: remove unneeded .encode()
astuyve Jun 23, 2022
cf376df
feat: revert setup
astuyve Jun 23, 2022
0978935
feat: Move encode to after nil check
astuyve Jun 23, 2022
b8c4460
feat: Guard against unset _last_export. Add tests for overriden perio…
astuyve Jun 27, 2022
4afe52c
feat: formatting
astuyve Jun 27, 2022
754367c
feat: release notes
astuyve Jun 27, 2022
d3e5a4c
feat: Add serverless to spelling list
astuyve Jun 27, 2022
79c2f23
feat: Incorporate feedback from JD
astuyve Jul 18, 2022
94cd7f2
feat: CR feedback from JD
astuyve Jul 18, 2022
3e5c9e4
lint: fix
astuyve Jul 18, 2022
811cb04
feat: remove unused import
astuyve Jul 18, 2022
1ab1ef3
feat: Human-readable release note
astuyve Jul 18, 2022
baadc76
feat: CR feedback to remove unneeded checks on periodic calls and red…
astuyve Aug 2, 2022
c4ed52e
feat: riot
astuyve Aug 2, 2022
e6845bf
wip: skipping assignment type check, going to ask JD for help
astuyve Aug 2, 2022
3df5933
feat: fmt
astuyve Aug 2, 2022
9109718
feat: re-add counter check to ensure we don't flush early.
astuyve Aug 3, 2022
5282916
feat: riot fmt
astuyve Aug 3, 2022
ea30d06
feat: remove unused import
astuyve Aug 3, 2022
77c133b
feat: fix bad rename
astuyve Aug 3, 2022
b64b3eb
feat: more format
astuyve Aug 3, 2022
aededd5
feat: Override reset interval after flushing
astuyve Aug 4, 2022
e600cbe
try fixing typing
jd Aug 5, 2022
662f681
move serverless into scheduler
jd Aug 5, 2022
528e325
use some var
jd Aug 5, 2022
6e2ab1d
test(serverless): add profiler test
jd Aug 5, 2022
57eef9c
fix profile test case
jd Aug 5, 2022
e07955f
Merge branch '1.x' into aj/serverless-profiling
mergify[bot] Aug 5, 2022
fc2c22b
feat: more CR feedback. Flipping implicit none checks to explicit, us…
astuyve Aug 8, 2022
3cbc43b
Merge branch '1.x' into aj/serverless-profiling
astuyve Aug 8, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions ddtrace/profiling/profiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,14 @@ class _ProfilerInstance(service.Service):

_recorder = attr.ib(init=False, default=None)
_collectors = attr.ib(init=False, default=None)
_scheduler = attr.ib(init=False, default=None)
_scheduler = attr.ib(
init=False,
default=None,
type=scheduler.Scheduler,
)
_lambda_function_name = attr.ib(
init=False, factory=lambda: os.environ.get("AWS_LAMBDA_FUNCTION_NAME"), type=Optional[str]
)

ENDPOINT_TEMPLATE = "https://intake.profile.{}"

Expand Down Expand Up @@ -165,6 +172,9 @@ def _build_default_exporters(self):
# to the agent base path.
endpoint_path = "profiling/v1/input"

if self._lambda_function_name is not None:
self.tags.update({"functionname": self._lambda_function_name.encode("utf-8")})

return [
http.PprofHTTPExporter(
service=self.service,
Expand Down Expand Up @@ -209,9 +219,11 @@ def __attrs_post_init__(self):
exporters = self._build_default_exporters()

if exporters:
self._scheduler = scheduler.Scheduler(
recorder=r, exporters=exporters, before_flush=self._collectors_snapshot
)
if self._lambda_function_name is None:
scheduler_class = scheduler.Scheduler
else:
scheduler_class = scheduler.ServerlessScheduler
self._scheduler = scheduler_class(recorder=r, exporters=exporters, before_flush=self._collectors_snapshot)

self.set_asyncio_event_loop_policy()

Expand Down
32 changes: 32 additions & 0 deletions ddtrace/profiling/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,35 @@ def periodic(self):
self.flush()
finally:
self.interval = max(0, self._configured_interval - (compat.monotonic() - start_time))


@attr.s
class ServerlessScheduler(Scheduler):
"""Serverless scheduler that works on, e.g., AWS Lambda.

The idea with this scheduler is to not sleep 60s, but to sleep 1s and flush out profiles after 60 sleeping period.
As the service can be frozen a few seconds after flushing out a profile, we want to make sure the next flush is not
> 60s later, but after at least 60 periods of 1s.

"""

# We force this interval everywhere
FORCED_INTERVAL = 1.0
FLUSH_AFTER_INTERVALS = 60.0

_interval = attr.ib(default=FORCED_INTERVAL, type=float)
_profiled_intervals = attr.ib(init=False, default=0)

def periodic(self):
# Check both the number of intervals and time frame to be sure we don't flush, e.g., empty profiles
if self._profiled_intervals >= self.FLUSH_AFTER_INTERVALS and (compat.time_ns() - self._last_export) >= (
self.FORCED_INTERVAL * self.FLUSH_AFTER_INTERVALS
):
try:
super(ServerlessScheduler, self).periodic()
finally:
# Override interval so it's always back to the value we n
self.interval = self.FORCED_INTERVAL
self._profiled_intervals = 0
else:
self._profiled_intervals += 1
2 changes: 2 additions & 0 deletions docs/spelling_wordlist.txt
Original file line number Diff line number Diff line change
Expand Up @@ -186,3 +186,5 @@ yaaredis
Kinesis
AppSec
libddwaf
Serverless
serverless
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
---
features:
- |
Adds support for Lambda profiling, which can be enabled by starting the profiler outside of the handler (on cold start).
9 changes: 9 additions & 0 deletions tests/profiling/test_profiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from ddtrace.profiling import event
from ddtrace.profiling import exporter
from ddtrace.profiling import profiler
from ddtrace.profiling import scheduler
from ddtrace.profiling.collector import asyncio
from ddtrace.profiling.collector import memalloc
from ddtrace.profiling.collector import stack
Expand Down Expand Up @@ -378,3 +379,11 @@ def test_default_collectors():
else:
assert any(isinstance(c, asyncio.AsyncioLockCollector) for c in p._profiler._collectors)
p.stop(flush=False)


def test_profiler_serverless(monkeypatch):
# type: (...) -> None
monkeypatch.setenv("AWS_LAMBDA_FUNCTION_NAME", "foobar")
p = profiler.Profiler()
assert isinstance(p._scheduler, scheduler.ServerlessScheduler)
assert p.tags["functionname"] == b"foobar"
20 changes: 20 additions & 0 deletions tests/profiling/test_scheduler.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
# -*- encoding: utf-8 -*-
import logging

import mock

from ddtrace.internal import compat
from ddtrace.profiling import event
from ddtrace.profiling import exporter
from ddtrace.profiling import recorder
Expand Down Expand Up @@ -54,3 +57,20 @@ def call_me():
assert caplog.record_tuples == [
(("ddtrace.profiling.scheduler", logging.ERROR, "Scheduler before_flush hook failed"))
]


@mock.patch("ddtrace.profiling.scheduler.Scheduler.periodic")
def test_serverless_periodic(mock_periodic):
r = recorder.Recorder()
s = scheduler.ServerlessScheduler(r, [exporter.NullExporter()])
# Fake start()
s._last_export = compat.time_ns()
s.periodic()
assert s._profiled_intervals == 1
mock_periodic.assert_not_called()
s._last_export = compat.time_ns() - 65
s._profiled_intervals = 65
s.periodic()
assert s._profiled_intervals == 0
assert s.interval == 1
mock_periodic.assert_called()