Skip to content

sdk-core: reduce breadcrumbs size #228

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Sep 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/node/src/BacktraceClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration>
FileBreadcrumbsStorage.create(
this.sessionFiles,
fileSystem,
clientSetup.options.breadcrumbs?.maximumBreadcrumbs ?? 100,
(clientSetup.options.breadcrumbs?.maximumBreadcrumbs ?? 100) || 100,
),
);
}
Expand Down
2 changes: 1 addition & 1 deletion packages/react-native/src/BacktraceClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration>
FileBreadcrumbsStorage.create(
fileSystem,
this.sessionFiles,
clientSetup.options.breadcrumbs?.maximumBreadcrumbs ?? 100,
(clientSetup.options.breadcrumbs?.maximumBreadcrumbs ?? 100) || 100,
),
);
}
Expand Down
20 changes: 17 additions & 3 deletions packages/sdk-core/src/builder/BacktraceCoreClientBuilder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@ import { BacktraceReportSubmission } from '../model/http/BacktraceReportSubmissi
import { BacktraceRequestHandler } from '../model/http/BacktraceRequestHandler.js';
import { BacktraceModule } from '../modules/BacktraceModule.js';
import { BacktraceAttributeProvider } from '../modules/attribute/BacktraceAttributeProvider.js';
import { BreadcrumbsEventSubscriber, BreadcrumbsStorage } from '../modules/breadcrumbs/index.js';
import {
BreadcrumbsEventSubscriber,
BreadcrumbsStorage,
BreadcrumbsStorageFactory,
} from '../modules/breadcrumbs/index.js';
import { BacktraceStackTraceConverter } from '../modules/converter/index.js';
import { BacktraceSessionProvider } from '../modules/metrics/BacktraceSessionProvider.js';
import { MetricsQueue } from '../modules/metrics/MetricsQueue.js';
Expand Down Expand Up @@ -39,12 +43,22 @@ export abstract class BacktraceCoreClientBuilder<S extends Partial<CoreClientSet
return this;
}

public useBreadcrumbsStorage(storage: BreadcrumbsStorage): this {
public useBreadcrumbsStorage(storageFactory: BreadcrumbsStorageFactory): this;
/**
* @deprecated Use `useBreadcrumbsStorage` with `BreadcrumbsStorageFactory`.
*/
public useBreadcrumbsStorage(storage: BreadcrumbsStorage): this;
public useBreadcrumbsStorage(storage: BreadcrumbsStorage | BreadcrumbsStorageFactory): this {
if (!this.clientSetup.breadcrumbsSetup) {
this.clientSetup.breadcrumbsSetup = {};
}

this.clientSetup.breadcrumbsSetup.storage = storage;
if (typeof storage === 'function') {
this.clientSetup.breadcrumbsSetup.storage = storage;
} else {
this.clientSetup.breadcrumbsSetup.storage = () => storage;
}

return this;
}

Expand Down
6 changes: 2 additions & 4 deletions packages/sdk-core/src/common/jsonEscaper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,10 @@ export function jsonEscaper() {
// in TypeScript add "this: any" param to avoid compliation errors - as follows
// return function (this: any, field: any, value: any) {
return function (this: unknown, key: string, value: unknown) {
if (!key) {
return value;
}
if (value === null) {
return value;
}

const valueType = typeof value;

if (valueType === 'bigint') {
Expand All @@ -28,7 +26,7 @@ export function jsonEscaper() {
keys.pop();
}
if (ancestors.includes(value)) {
return `[Circular].${keys.join('.')}.${key}`;
return `[Circular].${keys.filter((k) => !!k).join('.')}.${key}`;
}
keys.push(key);
ancestors.push(value);
Expand Down
137 changes: 137 additions & 0 deletions packages/sdk-core/src/common/jsonSize.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
type JsonReplacer = (this: unknown, key: string, value: unknown) => unknown;

function stringifiedSize<T>(value: T): number {
return JSON.stringify(value).length;
}

function toStringSize<T extends { toString(): string }>(value: T): number {
return value.toString().length;
}

const stringSize = (value: string) => stringifiedSize(value);
const numberSize = toStringSize<number>;
const bigintSize = toStringSize<bigint>;
const symbolSize = 0;
const functionSize = 0;
const booleanSize = (value: boolean) => (value ? 4 : 5);
const undefinedSize = 0;
const nullSize = 'null'.length;

function arraySize(array: unknown[], replacer?: JsonReplacer): number {
const bracketLength = 2;
const commaLength = array.length - 1;
let elementsLength = 0;
for (let i = 0; i < array.length; i++) {
const element = array[i];
switch (typeof element) {
case 'function':
case 'symbol':
case 'undefined':
elementsLength += nullSize;
break;
default:
elementsLength += _jsonSize(array, i.toString(), element, replacer);
}
}

return bracketLength + commaLength + elementsLength;
}

const objectSize = (obj: object, replacer?: JsonReplacer): number => {
const entries = Object.entries(obj);
const bracketLength = 2;

let entryCount = 0;
let entriesLength = 0;

for (const [k, v] of entries) {
const valueSize = _jsonSize(obj, k, v, replacer);
if (valueSize === 0) {
continue;
}

entryCount++;

// +1 adds the comma size
entriesLength += keySize(k) + valueSize + 1;
}

// -1 removes previously added last comma size (there is no trailing comma)
const commaLength = Math.max(0, entryCount - 1);

return bracketLength + commaLength + entriesLength;
};

function keySize(key: unknown): number {
const QUOTE_SIZE = 2;

if (key === null) {
return nullSize + QUOTE_SIZE;
} else if (key === undefined) {
return '"undefined"'.length;
}

switch (typeof key) {
case 'string':
return stringSize(key);
case 'number':
return numberSize(key) + QUOTE_SIZE;
case 'boolean':
return booleanSize(key) + QUOTE_SIZE;
case 'symbol':
return symbolSize; // key not used in JSON
default:
return stringSize(key.toString());
}
}

function _jsonSize(parent: unknown, key: string, value: unknown, replacer?: JsonReplacer): number {
if (value && typeof value === 'object' && 'toJSON' in value && typeof value.toJSON === 'function') {
value = value.toJSON() as object;
}

value = replacer ? replacer.call(parent, key, value) : value;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the operation order is incorrect. Otherwise you don't know in the replacer if you're dealing with the object or a JSON string. I think we should move L:93 just before object checks. Otherwise, can we left information why we need to execute code in this order?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure I understand. Can you give an example?
replacer has to be called there because 'should compute object size for self-referencing object with toJSON' test will fail due to stack overflow.

if (value === null) {
return nullSize;
} else if (value === undefined) {
return undefinedSize;
}

if (Array.isArray(value)) {
return arraySize(value, replacer);
}

switch (typeof value) {
case 'bigint':
return bigintSize(value);
case 'boolean':
return booleanSize(value);
case 'function':
return functionSize;
case 'number':
return numberSize(value);
case 'object':
return objectSize(value, replacer);
case 'string':
return stringSize(value);
case 'symbol':
return symbolSize;
case 'undefined':
return undefinedSize;
}

return 0;
}

/**
* Calculates size of the object as it would be serialized into JSON.
*
* _Should_ return the same value as `JSON.stringify(value, replacer).length`.
* This may not be 100% accurate, but should work for our requirements.
* @param value Value to compute length for.
* @param replacer A function that transforms the results as in `JSON.stringify`.
* @returns Final string length.
*/
export function jsonSize(value: unknown, replacer?: JsonReplacer): number {
return _jsonSize(undefined, '', value, replacer);
}
30 changes: 30 additions & 0 deletions packages/sdk-core/src/common/limitObjectDepth.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
type DeepPartial<T extends object> = Partial<{ [K in keyof T]: T[K] extends object ? DeepPartial<T[K]> : T[K] }>;

const REMOVED_PLACEHOLDER = '<removed>';

export type Limited<T extends object> = DeepPartial<T> | typeof REMOVED_PLACEHOLDER;

export function limitObjectDepth<T extends object>(obj: T, depth: number): Limited<T> {
if (!(depth < Infinity)) {
return obj;
}

if (depth < 0) {
return REMOVED_PLACEHOLDER;
}

const limitIfObject = (value: unknown) =>
typeof value === 'object' && value ? limitObjectDepth(value, depth - 1) : value;

const result: DeepPartial<T> = {};
for (const key in obj) {
const value = obj[key];
if (Array.isArray(value)) {
result[key] = value.map(limitIfObject) as never;
} else {
result[key] = limitIfObject(value) as never;
}
}

return result;
}
129 changes: 101 additions & 28 deletions packages/sdk-core/src/dataStructures/OverwritingArray.ts
Original file line number Diff line number Diff line change
@@ -1,47 +1,120 @@
import { OverwritingArrayIterator } from './OverwritingArrayIterator.js';
import { ConstrainedNumber, clamped, wrapped } from './numbers.js';

export class OverwritingArray<T> {
private _array: T[];
private _index = 0;
private _size = 0;
private _startIndex = 0;
constructor(public readonly capacity: number) {
this._array = this.createArray();

private readonly _headConstraint: ConstrainedNumber;
private readonly _lengthConstraint: ConstrainedNumber;

private _head = 0;
private _length = 0;

private get head() {
return this._head;
}

private set head(value: number) {
this._head = this._headConstraint(value);
}
public add(value: T): this {
this._array[this._index] = value;
this._index = this.incrementIndex(this._index);
this._startIndex = this.incrementStartingIndex();
this._size = this.incrementSize();
return this;

public get length() {
return this._length;
}

public clear(): void {
this._array = this.createArray();
public set length(value: number) {
this._length = this._lengthConstraint(value);
}

public values(): IterableIterator<T> {
return new OverwritingArrayIterator<T>(this._array, this._startIndex, this._size);
private get start() {
return this._headConstraint(this.head - this.length);
}

[Symbol.iterator](): IterableIterator<T> {
return new OverwritingArrayIterator<T>(this._array, this._startIndex, this._size);
constructor(
public readonly capacity: number,
items?: T[],
) {
this._array = new Array(capacity);

// Head must be always between 0 and capacity.
// If lower than 0, it needs to go from the end
// If larger than capacity, it needs to go from the start
// Wrapping solves this
this._headConstraint = wrapped(0, capacity);

// Length must be always no less than 0 and no larger than capacity
this._lengthConstraint = clamped(0, capacity);

if (items) {
this.push(...items);
}
}

private incrementIndex(index: number) {
return (index + 1) % this.capacity;
public add(item: T) {
return this.pushOne(item);
}

private incrementStartingIndex() {
if (this._size !== this.capacity) {
return this._startIndex;
public push(...items: T[]): number {
for (const item of items) {
this.pushOne(item);
}
return this.incrementIndex(this._startIndex);
return this.length;
}

public pop(): T | undefined {
this.head--;
const element = this._array[this.head];
this._array[this.head] = undefined as never;
this.length--;
return element;
}
private incrementSize() {
return Math.min(this.capacity, this._size + 1);

public shift(): T | undefined {
const element = this._array[this.start];
this._array[this.start] = undefined as never;
this.length--;
return element;
}

public at(index: number): T | undefined {
return this._array[this.index(index)];
}
private createArray() {
return new Array(this.capacity);

public *values(): IterableIterator<T> {
for (let i = 0; i < this.length; i++) {
const index = this.index(i);
yield this._array[index];
}
}

public *keys(): IterableIterator<number> {
for (let i = 0; i < this.length; i++) {
yield i;
}
}

public *entries(): IterableIterator<[number, T]> {
for (let i = 0; i < this.length; i++) {
const index = this.index(i);
yield [i, this._array[index]];
}
}

public [Symbol.iterator]() {
return this.values();
}

private pushOne(item: T) {
this._array[this.head] = item;
this.head++;
this.length++;
}

private index(value: number) {
if (!this.length) {
return this._headConstraint(value);
}

const index = (value % this.length) + this.start;
return this._headConstraint(index);
}
}
Loading