feat(core): Update metric normalization (getsentry#11518)

Closes getsentry#11433

https://develop.sentry.dev/sdk/metrics/#normalization

Author: timfish

packages/core/src/metrics/aggregator.ts

@@ -1,11 +1,11 @@
 import type { Client, MeasurementUnit, MetricsAggregator as MetricsAggregatorBase, Primitive } from '@sentry/types';
 import { timestampInSeconds } from '@sentry/utils';
 import { updateMetricSummaryOnActiveSpan } from '../utils/spanUtils';
-import { DEFAULT_FLUSH_INTERVAL, MAX_WEIGHT, NAME_AND_TAG_KEY_NORMALIZATION_REGEX, SET_METRIC_TYPE } from './constants';
+import { DEFAULT_FLUSH_INTERVAL, MAX_WEIGHT, SET_METRIC_TYPE } from './constants';
 import { captureAggregateMetrics } from './envelope';
 import { METRIC_MAP } from './instance';
 import type { MetricBucket, MetricType } from './types';
-import { getBucketKey, sanitizeTags } from './utils';
+import { getBucketKey, sanitizeMetricKey, sanitizeTags, sanitizeUnit } from './utils';
 
 /**
  * A metrics aggregator that aggregates metrics in memory and flushes them periodically.
@@ -46,6 +46,7 @@ export class MetricsAggregator implements MetricsAggregatorBase {
       // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
       this._interval.unref();
     }
+
     this._flushShift = Math.floor((Math.random() * DEFAULT_FLUSH_INTERVAL) / 1000);
     this._forceFlush = false;
   }
@@ -57,13 +58,14 @@ export class MetricsAggregator implements MetricsAggregatorBase {
     metricType: MetricType,
     unsanitizedName: string,
     value: number | string,
-    unit: MeasurementUnit = 'none',
+    unsanitizedUnit: MeasurementUnit = 'none',
     unsanitizedTags: Record<string, Primitive> = {},
     maybeFloatTimestamp = timestampInSeconds(),
   ): void {
     const timestamp = Math.floor(maybeFloatTimestamp);
-    const name = unsanitizedName.replace(NAME_AND_TAG_KEY_NORMALIZATION_REGEX, '_');
+    const name = sanitizeMetricKey(unsanitizedName);
     const tags = sanitizeTags(unsanitizedTags);
+    const unit = sanitizeUnit(unsanitizedUnit as string);
 
     const bucketKey = getBucketKey(metricType, name, unit, tags);
 

packages/core/src/metrics/browser-aggregator.ts

@@ -1,11 +1,11 @@
 import type { Client, MeasurementUnit, MetricsAggregator, Primitive } from '@sentry/types';
 import { timestampInSeconds } from '@sentry/utils';
 import { updateMetricSummaryOnActiveSpan } from '../utils/spanUtils';
-import { DEFAULT_BROWSER_FLUSH_INTERVAL, NAME_AND_TAG_KEY_NORMALIZATION_REGEX, SET_METRIC_TYPE } from './constants';
+import { DEFAULT_BROWSER_FLUSH_INTERVAL, SET_METRIC_TYPE } from './constants';
 import { captureAggregateMetrics } from './envelope';
 import { METRIC_MAP } from './instance';
 import type { MetricBucket, MetricType } from './types';
-import { getBucketKey, sanitizeTags } from './utils';
+import { getBucketKey, sanitizeMetricKey, sanitizeTags, sanitizeUnit } from './utils';
 
 /**
  * A simple metrics aggregator that aggregates metrics in memory and flushes them periodically.
@@ -32,13 +32,14 @@ export class BrowserMetricsAggregator implements MetricsAggregator {
     metricType: MetricType,
     unsanitizedName: string,
     value: number | string,
-    unit: MeasurementUnit | undefined = 'none',
+    unsanitizedUnit: MeasurementUnit | undefined = 'none',
     unsanitizedTags: Record<string, Primitive> | undefined = {},
     maybeFloatTimestamp: number | undefined = timestampInSeconds(),
   ): void {
     const timestamp = Math.floor(maybeFloatTimestamp);
-    const name = unsanitizedName.replace(NAME_AND_TAG_KEY_NORMALIZATION_REGEX, '_');
+    const name = sanitizeMetricKey(unsanitizedName);
     const tags = sanitizeTags(unsanitizedTags);
+    const unit = sanitizeUnit(unsanitizedUnit as string);
 
     const bucketKey = getBucketKey(metricType, name, unit, tags);
 
@@ -79,8 +80,7 @@ export class BrowserMetricsAggregator implements MetricsAggregator {
       return;
     }
 
-    // TODO(@anonrig): Use Object.values() when we support ES6+
-    const metricBuckets = Array.from(this._buckets).map(([, bucketItem]) => bucketItem);
+    const metricBuckets = Array.from(this._buckets.values());
     captureAggregateMetrics(this._client, metricBuckets);
 
     this._buckets.clear();

packages/core/src/metrics/constants.ts

@@ -3,26 +3,6 @@ export const GAUGE_METRIC_TYPE = 'g' as const;
 export const SET_METRIC_TYPE = 's' as const;
 export const DISTRIBUTION_METRIC_TYPE = 'd' as const;
 
-/**
- * Normalization regex for metric names and metric tag names.
- *
- * This enforces that names and tag keys only contain alphanumeric characters,
- * underscores, forward slashes, periods, and dashes.
- *
- * See: https://develop.sentry.dev/sdk/metrics/#normalization
- */
-export const NAME_AND_TAG_KEY_NORMALIZATION_REGEX = /[^a-zA-Z0-9_/.-]+/g;
-
-/**
- * Normalization regex for metric tag values.
- *
- * This enforces that values only contain words, digits, or the following
- * special characters: _:/@.{}[\]$-
- *
- * See: https://develop.sentry.dev/sdk/metrics/#normalization
- */
-export const TAG_VALUE_NORMALIZATION_REGEX = /[^\w\d\s_:/@.{}[\]$-]+/g;
-
 /**
  * This does not match spec in https://develop.sentry.dev/sdk/metrics
  * but was chosen to optimize for the most common case in browser environments.

packages/core/src/metrics/utils.ts

@@ -1,6 +1,5 @@
 import type { MeasurementUnit, MetricBucketItem, Primitive } from '@sentry/types';
 import { dropUndefinedKeys } from '@sentry/utils';
-import { NAME_AND_TAG_KEY_NORMALIZATION_REGEX, TAG_VALUE_NORMALIZATION_REGEX } from './constants';
 import type { MetricType } from './types';
 
 /**
@@ -54,15 +53,72 @@ export function serializeMetricBuckets(metricBucketItems: MetricBucketItem[]): s
   return out;
 }
 
+/**
+ * Sanitizes units
+ *
+ * These Regex's are straight from the normalisation docs:
+ * https://develop.sentry.dev/sdk/metrics/#normalization
+ */
+export function sanitizeUnit(unit: string): string {
+  return unit.replace(/[^\w]+/gi, '_');
+}
+
+/**
+ * Sanitizes metric keys
+ *
+ * These Regex's are straight from the normalisation docs:
+ * https://develop.sentry.dev/sdk/metrics/#normalization
+ */
+export function sanitizeMetricKey(key: string): string {
+  return key.replace(/[^\w\-.]+/gi, '_');
+}
+
+/**
+ * Sanitizes metric keys
+ *
+ * These Regex's are straight from the normalisation docs:
+ * https://develop.sentry.dev/sdk/metrics/#normalization
+ */
+function sanitizeTagKey(key: string): string {
+  return key.replace(/[^\w\-./]+/gi, '');
+}
+
+/**
+ * These Regex's are straight from the normalisation docs:
+ * https://develop.sentry.dev/sdk/metrics/#normalization
+ */
+const tagValueReplacements: [string, string][] = [
+  ['\n', '\\n'],
+  ['\r', '\\r'],
+  ['\t', '\\t'],
+  ['\\', '\\\\'],
+  ['|', '\\u{7c}'],
+  [',', '\\u{2c}'],
+];
+
+function getCharOrReplacement(input: string): string {
+  for (const [search, replacement] of tagValueReplacements) {
+    if (input === search) {
+      return replacement;
+    }
+  }
+
+  return input;
+}
+
+function sanitizeTagValue(value: string): string {
+  return [...value].reduce((acc, char) => acc + getCharOrReplacement(char), '');
+}
+
 /**
  * Sanitizes tags.
  */
 export function sanitizeTags(unsanitizedTags: Record<string, Primitive>): Record<string, string> {
   const tags: Record<string, string> = {};
   for (const key in unsanitizedTags) {
     if (Object.prototype.hasOwnProperty.call(unsanitizedTags, key)) {
-      const sanitizedKey = key.replace(NAME_AND_TAG_KEY_NORMALIZATION_REGEX, '_');
-      tags[sanitizedKey] = String(unsanitizedTags[key]).replace(TAG_VALUE_NORMALIZATION_REGEX, '');
+      const sanitizedKey = sanitizeTagKey(key);
+      tags[sanitizedKey] = sanitizeTagValue(String(unsanitizedTags[key]));
     }
   }
   return tags;

packages/core/test/lib/metrics/utils.test.ts

@@ -4,7 +4,7 @@ import {
   GAUGE_METRIC_TYPE,
   SET_METRIC_TYPE,
 } from '../../../src/metrics/constants';
-import { getBucketKey } from '../../../src/metrics/utils';
+import { getBucketKey, sanitizeTags } from '../../../src/metrics/utils';
 
 describe('getBucketKey', () => {
   it.each([
@@ -18,4 +18,26 @@ describe('getBucketKey', () => {
   ])('should return', (metricType, name, unit, tags, expected) => {
     expect(getBucketKey(metricType, name, unit, tags)).toEqual(expected);
   });
+
+  it('should sanitize tags', () => {
+    const inputTags = {
+      'f-oo|bar': '%$foo/',
+      'foo$.$.$bar': 'blah{}',
+      'foö-bar': 'snöwmän',
+      route: 'GET /foo',
+      __bar__: 'this | or , that',
+      'foo/': 'hello!\n\r\t\\',
+    };
+
+    const outputTags = {
+      'f-oobar': '%$foo/',
+      'foo..bar': 'blah{}',
+      'fo-bar': 'snöwmän',
+      route: 'GET /foo',
+      __bar__: 'this \\u{7c} or \\u{2c} that',
+      'foo/': 'hello!\\n\\r\\t\\\\',
+    };
+
+    expect(sanitizeTags(inputTags)).toEqual(outputTags);
+  });
 });