Skip to content

Commit 78c1900

Browse files
duncanistaduncanista
authored
chore: lazy load internal request module (#519)
* remove `post` and `get` exports from `metrics/index.ts` * switch `post` import from `utils/request` * remove local flush from extension code * lazy load `request` to remove 1 MB dep * update import from `extension` * rename `agent` to `extension` * update imports and move method to private function * update imports and add await * fix tests * move tests from extension to listener test flush * lint and format * update snapshots? * bring back local flush to `extension.ts` * update tests again * fix error which patched twice the http/https modules
1 parent e08ad86 commit 78c1900

File tree

10 files changed

+58
-45
lines changed

10 files changed

+58
-45
lines changed

‎integration_tests/snapshots/logs/sync-metrics_node16.log

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ XXXX-XX-XX XX:XX:XX.XXX INFO [dd.trace_id=XXXX dd.span_id=XXXX] Processed APIGat
1919
HTTP POST https://api.datadoghq.com/api/v1/distribution_points?api_key=XXXX TraceHeaders: ["x-datadog-parent-id:XXXX","x-datadog-sampling-priority:-1","x-datadog-trace-id:XXXX"]
2020
END Duration: XXXX ms (init: XXXX ms) Memory Used: XXXX MB
2121
START
22+
XXXX-XX-XX XX:XX:XX.XXX INFO [dd.trace_id=XXXX dd.span_id=XXXX] Processed SNS request
2223
{
2324
"e": XXXX,
2425
"m": "aws.lambda.enhanced.invocations",
@@ -34,10 +35,10 @@ START
3435
],
3536
"v": 1
3637
}
37-
XXXX-XX-XX XX:XX:XX.XXX INFO [dd.trace_id=XXXX dd.span_id=XXXX] Processed SNS request
3838
HTTP POST https://api.datadoghq.com/api/v1/distribution_points?api_key=XXXX TraceHeaders: ["x-datadog-parent-id:XXXX","x-datadog-sampling-priority:-1","x-datadog-trace-id:XXXX"]
3939
END Duration: XXXX ms Memory Used: XXXX MB
4040
START
41+
XXXX-XX-XX XX:XX:XX.XXX INFO [dd.trace_id=XXXX dd.span_id=XXXX] Processed SQS request
4142
{
4243
"e": XXXX,
4344
"m": "aws.lambda.enhanced.invocations",
@@ -53,6 +54,5 @@ START
5354
],
5455
"v": 1
5556
}
56-
XXXX-XX-XX XX:XX:XX.XXX INFO [dd.trace_id=XXXX dd.span_id=XXXX] Processed SQS request
5757
HTTP POST https://api.datadoghq.com/api/v1/distribution_points?api_key=XXXX TraceHeaders: ["x-datadog-parent-id:XXXX","x-datadog-sampling-priority:-1","x-datadog-trace-id:XXXX"]
5858
END Duration: XXXX ms Memory Used: XXXX MB

‎integration_tests/snapshots/logs/sync-metrics_node18.log

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ XXXX-XX-XX XX:XX:XX.XXX INFO [dd.trace_id=XXXX dd.span_id=XXXX] Processed APIGat
1919
HTTP POST https://api.datadoghq.com/api/v1/distribution_points?api_key=XXXX TraceHeaders: ["x-datadog-parent-id:XXXX","x-datadog-sampling-priority:-1","x-datadog-trace-id:XXXX"]
2020
END Duration: XXXX ms (init: XXXX ms) Memory Used: XXXX MB
2121
START
22+
XXXX-XX-XX XX:XX:XX.XXX INFO [dd.trace_id=XXXX dd.span_id=XXXX] Processed SNS request
2223
{
2324
"e": XXXX,
2425
"m": "aws.lambda.enhanced.invocations",
@@ -34,7 +35,6 @@ START
3435
],
3536
"v": 1
3637
}
37-
XXXX-XX-XX XX:XX:XX.XXX INFO [dd.trace_id=XXXX dd.span_id=XXXX] Processed SNS request
3838
HTTP POST https://api.datadoghq.com/api/v1/distribution_points?api_key=XXXX TraceHeaders: ["x-datadog-parent-id:XXXX","x-datadog-sampling-priority:-1","x-datadog-trace-id:XXXX"]
3939
END Duration: XXXX ms Memory Used: XXXX MB
4040
START

‎integration_tests/snapshots/logs/sync-metrics_node20.log

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ XXXX-XX-XX XX:XX:XX.XXX INFO [dd.trace_id=XXXX dd.span_id=XXXX] Processed APIGat
1919
HTTP POST https://api.datadoghq.com/api/v1/distribution_points?api_key=XXXX TraceHeaders: ["x-datadog-parent-id:XXXX","x-datadog-sampling-priority:-1","x-datadog-trace-id:XXXX"]
2020
END Duration: XXXX ms (init: XXXX ms) Memory Used: XXXX MB
2121
START
22-
XXXX-XX-XX XX:XX:XX.XXX INFO [dd.trace_id=XXXX dd.span_id=XXXX] Processed SNS request
2322
{
2423
"e": XXXX,
2524
"m": "aws.lambda.enhanced.invocations",
@@ -35,9 +34,11 @@ XXXX-XX-XX XX:XX:XX.XXX INFO [dd.trace_id=XXXX dd.span_id=XXXX] Processed SNS re
3534
],
3635
"v": 1
3736
}
37+
XXXX-XX-XX XX:XX:XX.XXX INFO [dd.trace_id=XXXX dd.span_id=XXXX] Processed SNS request
3838
HTTP POST https://api.datadoghq.com/api/v1/distribution_points?api_key=XXXX TraceHeaders: ["x-datadog-parent-id:XXXX","x-datadog-sampling-priority:-1","x-datadog-trace-id:XXXX"]
3939
END Duration: XXXX ms Memory Used: XXXX MB
4040
START
41+
XXXX-XX-XX XX:XX:XX.XXX INFO [dd.trace_id=XXXX dd.span_id=XXXX] Processed SQS request
4142
{
4243
"e": XXXX,
4344
"m": "aws.lambda.enhanced.invocations",
@@ -53,6 +54,5 @@ START
5354
],
5455
"v": 1
5556
}
56-
XXXX-XX-XX XX:XX:XX.XXX INFO [dd.trace_id=XXXX dd.span_id=XXXX] Processed SQS request
5757
HTTP POST https://api.datadoghq.com/api/v1/distribution_points?api_key=XXXX TraceHeaders: ["x-datadog-parent-id:XXXX","x-datadog-sampling-priority:-1","x-datadog-trace-id:XXXX"]
5858
END Duration: XXXX ms Memory Used: XXXX MB

‎src/metrics/api.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import querystring from "querystring";
22
import { URL } from "url";
33

4-
import { logDebug, post } from "../utils";
4+
import { logDebug } from "../utils";
5+
import { post } from "../utils/request";
56
import { APIMetric } from "./model";
67

78
const API_KEY_QUERY_PARAM = "api_key";

‎src/metrics/extension.spec.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import nock from "nock";
2-
3-
import { isExtensionRunning, flushExtension, EXTENSION_URL } from "./extension";
2+
import { isExtensionRunning, EXTENSION_URL, flushExtension } from "./extension";
43
import mock from "mock-fs";
54

65
describe("isExtensionRunning", () => {
@@ -22,15 +21,16 @@ describe("isExtensionRunning", () => {
2221
expect(ran).toBeFalsy();
2322
});
2423
});
24+
2525
describe("flushExtension", () => {
2626
it("calls flush on the extension", async () => {
2727
const scope = nock(EXTENSION_URL).post("/lambda/flush", JSON.stringify({})).reply(200);
28-
await flushExtension();
28+
await flushExtension(true);
2929
expect(scope.isDone()).toBeTruthy();
3030
});
3131
it("catches error when flush doesn't respond", async () => {
3232
const scope = nock(EXTENSION_URL).post("/lambda/flush", JSON.stringify({})).replyWithError("Unavailable");
33-
await flushExtension();
33+
await flushExtension(true);
3434
expect(scope.isDone()).toBeTruthy();
3535
});
3636
});

‎src/metrics/extension.ts

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,16 @@
11
import { URL } from "url";
2-
import { post, logDebug, logError } from "../utils";
2+
import { logDebug, logError } from "../utils";
33
import fs from "fs";
44

55
export const EXTENSION_URL = "http://127.0.0.1:8124";
66
const EXTENSION_PATH = "/opt/extensions/datadog-agent";
7-
const LOCAL_FLUSH_PATH = "/lambda/flush";
87
const LOCAL_FLUSH_TIMEOUT_MS = 100;
8+
const LOCAL_FLUSH_PATH = "/lambda/flush";
99

1010
export async function isExtensionRunning() {
1111
const extensionExists = await fileExists(EXTENSION_PATH);
1212
if (!extensionExists) {
13-
logDebug(`Extension Layer is not present`);
14-
return false;
15-
}
16-
return true;
17-
}
18-
19-
export async function flushExtension(): Promise<boolean> {
20-
const url = new URL(LOCAL_FLUSH_PATH, EXTENSION_URL);
21-
const result = await post(url, {}, { timeout: LOCAL_FLUSH_TIMEOUT_MS });
22-
if (!result.success) {
23-
logError(`Failed to flush extension. ${result.errorMessage}`);
13+
logDebug(`Extension Layer is not present.`);
2414
return false;
2515
}
2616
return true;
@@ -32,3 +22,24 @@ function fileExists(filename: string): Promise<boolean> {
3222
.then(() => true)
3323
.catch(() => false);
3424
}
25+
26+
export async function flushExtension(localTesting = false) {
27+
if (localTesting) {
28+
try {
29+
const { post } = require("../utils/request");
30+
const url = new URL(LOCAL_FLUSH_PATH, EXTENSION_URL);
31+
const result = await post(url, {}, { timeout: LOCAL_FLUSH_TIMEOUT_MS });
32+
if (!result.success) {
33+
logError(`Failed to flush extension. ${result.errorMessage}`);
34+
}
35+
36+
return true;
37+
} catch (error) {
38+
if (error instanceof Error) {
39+
logError("Failed to flush extension", error);
40+
}
41+
}
42+
}
43+
44+
return false;
45+
}

‎src/metrics/listener.spec.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ describe("MetricsListener", () => {
7878
siteURL,
7979
});
8080

81-
listener.onStartInvocation({});
81+
await listener.onStartInvocation({});
8282
listener.sendDistributionMetric("my-metric", 10, false, "tag:a", "tag:b");
8383
await expect(listener.onCompleteInvocation()).resolves.toEqual(undefined);
8484
});
@@ -153,7 +153,7 @@ describe("MetricsListener", () => {
153153
});
154154
// jest.useFakeTimers();
155155

156-
listener.onStartInvocation({});
156+
await listener.onStartInvocation({});
157157
listener.sendDistributionMetricWithDate("my-metric", 10, new Date(1584983836 * 1000), false, "tag:a", "tag:b");
158158
await listener.onCompleteInvocation();
159159

‎src/metrics/listener.ts

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
11
import { StatsD } from "hot-shots";
22
import { promisify } from "util";
33
import { logDebug, logError } from "../utils";
4-
import { APIClient } from "./api";
54
import { flushExtension, isExtensionRunning } from "./extension";
65
import { KMSService } from "./kms-service";
76
import { writeMetricToStdout } from "./metric-log";
87
import { Distribution } from "./model";
9-
import { Processor } from "./processor";
108

11-
const metricsBatchSendIntervalMS = 10000; // 10 seconds
9+
const METRICS_BATCH_SEND_INTERVAL = 10000; // 10 seconds
1210

1311
export interface MetricsConfig {
1412
/**
@@ -56,7 +54,7 @@ export interface MetricsConfig {
5654
}
5755

5856
export class MetricsListener {
59-
private currentProcessor?: Promise<Processor>;
57+
private currentProcessor?: Promise<any>;
6058
private apiKey: Promise<string>;
6159
private statsDClient?: StatsD;
6260
private isExtensionRunning?: boolean = undefined;
@@ -83,6 +81,7 @@ export class MetricsListener {
8381

8482
return;
8583
}
84+
8685
this.currentProcessor = this.createProcessor(this.config, this.apiKey);
8786
}
8887

@@ -120,15 +119,11 @@ export class MetricsListener {
120119
logError("failed to flush metrics", error as Error);
121120
}
122121
}
123-
try {
124-
if (this.isExtensionRunning && this.config.localTesting) {
125-
logDebug(`Flushing Extension for local test`);
126-
await flushExtension();
127-
}
128-
} catch (error) {
129-
if (error instanceof Error) {
130-
logError("failed to flush extension", error as Error);
131-
}
122+
123+
// Flush only when testing extension locally.
124+
// Passing config flag so we can lazy load the request module.
125+
if (this.isExtensionRunning) {
126+
await flushExtension(this.config.localTesting);
132127
}
133128
this.currentProcessor = undefined;
134129
}
@@ -171,12 +166,17 @@ export class MetricsListener {
171166
}
172167

173168
private async createProcessor(config: MetricsConfig, apiKey: Promise<string>) {
174-
const key = await apiKey;
175-
const url = `https://api.${config.siteURL}`;
176-
const apiClient = new APIClient(key, url);
177-
const processor = new Processor(apiClient, metricsBatchSendIntervalMS, config.shouldRetryMetrics);
178-
processor.startProcessing();
179-
return processor;
169+
if (!this.isExtensionRunning && !this.config.logForwarding) {
170+
const { APIClient } = require("./api");
171+
const { Processor } = require("./processor");
172+
173+
const key = await apiKey;
174+
const url = `https://api.${config.siteURL}`;
175+
const apiClient = new APIClient(key, url);
176+
const processor = new Processor(apiClient, METRICS_BATCH_SEND_INTERVAL, config.shouldRetryMetrics);
177+
processor.startProcessing();
178+
return processor;
179+
}
180180
}
181181

182182
private async getAPIKey(config: MetricsConfig) {

‎src/trace/patch-http.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ export function unpatchHttp() {
3333
}
3434

3535
function patchMethod(mod: typeof http | typeof https, method: "get" | "request", contextService: TraceContextService) {
36+
if (mod[method].__wrapped !== undefined) return; // Only patch once
37+
3638
shimmer.wrap(mod, method, (original) => {
3739
const fn = (arg1: any, arg2: any, arg3: any) => {
3840
[arg1, arg2, arg3] = addTraceContextToArgs(contextService, arg1, arg2, arg3);

‎src/utils/index.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,4 @@ export { didFunctionColdStart, getSandboxInitTags, setSandboxInit, isProactiveIn
22
export { wrap, promisifiedHandler } from "./handler";
33
export { Timer } from "./timer";
44
export { logError, logDebug, Logger, setLogLevel, setLogger, LogLevel } from "./log";
5-
export { get, post } from "./request";
65
export { tagObject } from "./tag-object";

0 commit comments

Comments
 (0)