From 351bbe571796b3b010841aadacf597d48b8f8506 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Thu, 27 Oct 2022 16:38:31 +0200 Subject: [PATCH 1/4] feat(otel): Capture otel attributes/resources as context --- .../opentelemetry-node/src/spanprocessor.ts | 25 ++++--- .../test/spanprocessor.test.ts | 68 ++++++++++++++++++- 2 files changed, 83 insertions(+), 10 deletions(-) diff --git a/packages/opentelemetry-node/src/spanprocessor.ts b/packages/opentelemetry-node/src/spanprocessor.ts index 35b52ea175ea..f51a1f95a6c2 100644 --- a/packages/opentelemetry-node/src/spanprocessor.ts +++ b/packages/opentelemetry-node/src/spanprocessor.ts @@ -1,6 +1,7 @@ import { Context } from '@opentelemetry/api'; import { Span as OtelSpan, SpanProcessor as OtelSpanProcessor } from '@opentelemetry/sdk-trace-base'; -import { getCurrentHub } from '@sentry/core'; +import { getCurrentHub, setContext } from '@sentry/core'; +import { Transaction } from '@sentry/tracing'; import { Span as SentrySpan, TransactionContext } from '@sentry/types'; import { logger } from '@sentry/utils'; @@ -65,18 +66,20 @@ export class SentrySpanProcessor implements OtelSpanProcessor { */ public onEnd(otelSpan: OtelSpan): void { const otelSpanId = otelSpan.spanContext().spanId; - const mapVal = this._map.get(otelSpanId); + const sentrySpan = this._map.get(otelSpanId); - if (!mapVal) { + if (!sentrySpan) { __DEBUG_BUILD__ && logger.error(`SentrySpanProcessor could not find span with OTEL-spanId ${otelSpanId} to finish.`); return; } - const sentrySpan = mapVal; - - // TODO: actually add context etc. to span - // updateSpanWithOtelData(sentrySpan, otelSpan); + if (sentrySpan instanceof Transaction) { + updateContextWithOtelData(otelSpan); + } else { + // TODO: + // updateSpanWithOtelData(sentrySpan, otelSpan); + } sentrySpan.finish(otelSpan.endTime[0]); @@ -111,5 +114,9 @@ function getTraceData(otelSpan: OtelSpan): Partial { return { spanId, traceId, parentSpanId }; } -// function updateSpanWithOtelData(sentrySpan: SentrySpan, otelSpan: OtelSpan): void { -// } +function updateContextWithOtelData(otelSpan: OtelSpan): void { + setContext('otel', { + attributes: otelSpan.attributes, + resource: otelSpan.resource.attributes, + }); +} diff --git a/packages/opentelemetry-node/test/spanprocessor.test.ts b/packages/opentelemetry-node/test/spanprocessor.test.ts index ec8947996c84..2419d52d0561 100644 --- a/packages/opentelemetry-node/test/spanprocessor.test.ts +++ b/packages/opentelemetry-node/test/spanprocessor.test.ts @@ -1,8 +1,11 @@ import * as OpenTelemetry from '@opentelemetry/api'; +import { Resource } from '@opentelemetry/resources'; import { Span as OtelSpan } from '@opentelemetry/sdk-trace-base'; import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; +import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import { Hub, makeMain } from '@sentry/core'; import { addExtensionMethods, Span as SentrySpan, Transaction } from '@sentry/tracing'; +import { Contexts, Scope } from '@sentry/types'; import { SentrySpanProcessor } from '../src/spanprocessor'; @@ -22,7 +25,11 @@ describe('SentrySpanProcessor', () => { makeMain(hub); spanProcessor = new SentrySpanProcessor(); - provider = new NodeTracerProvider(); + provider = new NodeTracerProvider({ + resource: new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: 'test-service', + }), + }); provider.addSpanProcessor(spanProcessor); provider.register(); }); @@ -36,6 +43,11 @@ describe('SentrySpanProcessor', () => { return spanProcessor._map.get(otelSpan.spanContext().spanId); } + function getContext() { + const scope = hub.getScope() as unknown as Scope & { _contexts: Contexts }; + return scope._contexts; + } + it('creates a transaction', async () => { const startTime = otelNumberToHrtime(new Date().valueOf()); @@ -125,6 +137,60 @@ describe('SentrySpanProcessor', () => { parentOtelSpan.end(); }); }); + + it('sets context for transaction', async () => { + const otelSpan = provider.getTracer('default').startSpan('GET /users'); + + // context is only set after end + expect(getContext()).toEqual({}); + + otelSpan.end(); + + expect(getContext()).toEqual({ + otel: { + attributes: {}, + resource: { + 'service.name': 'test-service', + 'telemetry.sdk.language': 'nodejs', + 'telemetry.sdk.name': 'opentelemetry', + 'telemetry.sdk.version': '1.7.0', + }, + }, + }); + + // Start new transaction, context should remain the same + const otelSpan2 = provider.getTracer('default').startSpan('GET /companies'); + + expect(getContext()).toEqual({ + otel: { + attributes: {}, + resource: { + 'service.name': 'test-service', + 'telemetry.sdk.language': 'nodejs', + 'telemetry.sdk.name': 'opentelemetry', + 'telemetry.sdk.version': '1.7.0', + }, + }, + }); + + otelSpan2.setAttribute('test-attribute', 'test-value'); + + otelSpan2.end(); + + expect(getContext()).toEqual({ + otel: { + attributes: { + 'test-attribute': 'test-value', + }, + resource: { + 'service.name': 'test-service', + 'telemetry.sdk.language': 'nodejs', + 'telemetry.sdk.name': 'opentelemetry', + 'telemetry.sdk.version': '1.7.0', + }, + }, + }); + }); }); // OTEL expects a custom date format From 3b365a007988bf3e4787805733aab001dc74f91d Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Fri, 28 Oct 2022 10:35:36 +0200 Subject: [PATCH 2/4] feat(otel): Capture otel attributes/kind as span data --- .../opentelemetry-node/src/spanprocessor.ts | 15 ++++++++-- .../test/spanprocessor.test.ts | 29 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/packages/opentelemetry-node/src/spanprocessor.ts b/packages/opentelemetry-node/src/spanprocessor.ts index f51a1f95a6c2..068500653980 100644 --- a/packages/opentelemetry-node/src/spanprocessor.ts +++ b/packages/opentelemetry-node/src/spanprocessor.ts @@ -77,8 +77,7 @@ export class SentrySpanProcessor implements OtelSpanProcessor { if (sentrySpan instanceof Transaction) { updateContextWithOtelData(otelSpan); } else { - // TODO: - // updateSpanWithOtelData(sentrySpan, otelSpan); + updateSpanWithOtelData(sentrySpan, otelSpan); } sentrySpan.finish(otelSpan.endTime[0]); @@ -120,3 +119,15 @@ function updateContextWithOtelData(otelSpan: OtelSpan): void { resource: otelSpan.resource.attributes, }); } + +function updateSpanWithOtelData(sentrySpan: SentrySpan, otelSpan: OtelSpan): void { + const { attributes, kind } = otelSpan; + + // TODO: Set status + sentrySpan.setData('otel.kind', kind.valueOf()); + + Object.keys(attributes).forEach(prop => { + const value = attributes[prop]; + sentrySpan.setData(prop, value); + }); +} diff --git a/packages/opentelemetry-node/test/spanprocessor.test.ts b/packages/opentelemetry-node/test/spanprocessor.test.ts index 2419d52d0561..0d41715dff30 100644 --- a/packages/opentelemetry-node/test/spanprocessor.test.ts +++ b/packages/opentelemetry-node/test/spanprocessor.test.ts @@ -191,6 +191,35 @@ describe('SentrySpanProcessor', () => { }, }); }); + + it('sets data for span', async () => { + const tracer = provider.getTracer('default'); + + tracer.startActiveSpan('GET /users', parentOtelSpan => { + tracer.startActiveSpan('SELECT * FROM users;', child => { + child.setAttribute('test-attribute', 'test-value'); + child.setAttribute('test-attribute-2', [1, 2, 3]); + child.setAttribute('test-attribute-3', 0); + child.setAttribute('test-attribute-4', false); + + const sentrySpan = getSpanForOtelSpan(child); + + expect(sentrySpan?.data).toEqual({}); + + child.end(); + + expect(sentrySpan?.data).toEqual({ + 'otel.kind': 0, + 'test-attribute': 'test-value', + 'test-attribute-2': [1, 2, 3], + 'test-attribute-3': 0, + 'test-attribute-4': false, + }); + }); + + parentOtelSpan.end(); + }); + }); }); // OTEL expects a custom date format From 1f0081abf8eb0b87d74ca8702752a5faa62df7fd Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Fri, 28 Oct 2022 11:51:15 +0200 Subject: [PATCH 3/4] feat(otel): Set sentry span status based on otel span --- packages/opentelemetry-node/package.json | 3 +- .../opentelemetry-node/src/spanprocessor.ts | 9 +- .../src/utils/map-otel-status.ts | 77 ++++++++++++++ .../test/spanprocessor.test.ts | 100 +++++++++++++++++- 4 files changed, 185 insertions(+), 4 deletions(-) create mode 100644 packages/opentelemetry-node/src/utils/map-otel-status.ts diff --git a/packages/opentelemetry-node/package.json b/packages/opentelemetry-node/package.json index 54318933a576..5af61a07f7a4 100644 --- a/packages/opentelemetry-node/package.json +++ b/packages/opentelemetry-node/package.json @@ -24,7 +24,8 @@ }, "peerDependencies": { "@opentelemetry/api": "1.x", - "@opentelemetry/sdk-trace-base": "1.x" + "@opentelemetry/sdk-trace-base": "1.x", + "@opentelemetry/semantic-conventions": "1.x" }, "devDependencies": { "@opentelemetry/api": "^1.2.0", diff --git a/packages/opentelemetry-node/src/spanprocessor.ts b/packages/opentelemetry-node/src/spanprocessor.ts index 068500653980..0fd37e1835fa 100644 --- a/packages/opentelemetry-node/src/spanprocessor.ts +++ b/packages/opentelemetry-node/src/spanprocessor.ts @@ -5,6 +5,8 @@ import { Transaction } from '@sentry/tracing'; import { Span as SentrySpan, TransactionContext } from '@sentry/types'; import { logger } from '@sentry/utils'; +import { mapOtelStatus } from './utils/map-otel-status'; + /** * Converts OpenTelemetry Spans to Sentry Spans and sends them to Sentry via * the Sentry SDK. @@ -76,6 +78,7 @@ export class SentrySpanProcessor implements OtelSpanProcessor { if (sentrySpan instanceof Transaction) { updateContextWithOtelData(otelSpan); + updateTransactionWithOtelData(sentrySpan, otelSpan); } else { updateSpanWithOtelData(sentrySpan, otelSpan); } @@ -123,7 +126,7 @@ function updateContextWithOtelData(otelSpan: OtelSpan): void { function updateSpanWithOtelData(sentrySpan: SentrySpan, otelSpan: OtelSpan): void { const { attributes, kind } = otelSpan; - // TODO: Set status + sentrySpan.setStatus(mapOtelStatus(otelSpan)); sentrySpan.setData('otel.kind', kind.valueOf()); Object.keys(attributes).forEach(prop => { @@ -131,3 +134,7 @@ function updateSpanWithOtelData(sentrySpan: SentrySpan, otelSpan: OtelSpan): voi sentrySpan.setData(prop, value); }); } + +function updateTransactionWithOtelData(transaction: Transaction, otelSpan: OtelSpan): void { + transaction.setStatus(mapOtelStatus(otelSpan)); +} diff --git a/packages/opentelemetry-node/src/utils/map-otel-status.ts b/packages/opentelemetry-node/src/utils/map-otel-status.ts new file mode 100644 index 000000000000..aca20abd961a --- /dev/null +++ b/packages/opentelemetry-node/src/utils/map-otel-status.ts @@ -0,0 +1,77 @@ +import { Span as OtelSpan } from '@opentelemetry/sdk-trace-base'; +import { SemanticAttributes } from '@opentelemetry/semantic-conventions'; +import { SpanStatusType as SentryStatus } from '@sentry/tracing'; + +// canonicalCodesHTTPMap maps some HTTP codes to Sentry's span statuses. See possible mapping in https://develop.sentry.dev/sdk/event-payloads/span/ +const canonicalCodesHTTPMap: Record = { + '400': 'failed_precondition', + '401': 'unauthenticated', + '403': 'permission_denied', + '404': 'not_found', + '409': 'aborted', + '429': 'resource_exhausted', + '499': 'cancelled', + '500': 'internal_error', + '501': 'unimplemented', + '503': 'unavailable', + '504': 'deadline_exceeded', +} as const; + +// canonicalCodesGrpcMap maps some GRPC codes to Sentry's span statuses. See description in grpc documentation. +const canonicalCodesGrpcMap: Record = { + '1': 'cancelled', + '2': 'unknown_error', + '3': 'invalid_argument', + '4': 'deadline_exceeded', + '5': 'not_found', + '6': 'already_exists', + '7': 'permission_denied', + '8': 'resource_exhausted', + '9': 'failed_precondition', + '10': 'aborted', + '11': 'out_of_range', + '12': 'unimplemented', + '13': 'internal_error', + '14': 'unavailable', + '15': 'data_loss', + '16': 'unauthenticated', +} as const; + +/** + * Get a Sentry span status from an otel span. + * + * @param otelSpan An otel span to generate a sentry status for. + * @returns The Sentry span status + */ +export function mapOtelStatus(otelSpan: OtelSpan): SentryStatus { + const { status, attributes } = otelSpan; + + const statusCode = status.code; + + if (statusCode < 0 || statusCode > 2) { + return 'unknown_error'; + } + + if (statusCode === 0 || statusCode === 1) { + return 'ok'; + } + + const httpCode = attributes[SemanticAttributes.HTTP_STATUS_CODE]; + const grpcCode = attributes[SemanticAttributes.RPC_GRPC_STATUS_CODE]; + + if (typeof httpCode === 'string') { + const sentryStatus = canonicalCodesHTTPMap[httpCode]; + if (sentryStatus) { + return sentryStatus; + } + } + + if (typeof grpcCode === 'string') { + const sentryStatus = canonicalCodesGrpcMap[grpcCode]; + if (sentryStatus) { + return sentryStatus; + } + } + + return 'unknown_error'; +} diff --git a/packages/opentelemetry-node/test/spanprocessor.test.ts b/packages/opentelemetry-node/test/spanprocessor.test.ts index 0d41715dff30..8a48ece8c428 100644 --- a/packages/opentelemetry-node/test/spanprocessor.test.ts +++ b/packages/opentelemetry-node/test/spanprocessor.test.ts @@ -2,9 +2,9 @@ import * as OpenTelemetry from '@opentelemetry/api'; import { Resource } from '@opentelemetry/resources'; import { Span as OtelSpan } from '@opentelemetry/sdk-trace-base'; import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; -import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; +import { SemanticAttributes, SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import { Hub, makeMain } from '@sentry/core'; -import { addExtensionMethods, Span as SentrySpan, Transaction } from '@sentry/tracing'; +import { addExtensionMethods, Span as SentrySpan, SpanStatusType, Transaction } from '@sentry/tracing'; import { Contexts, Scope } from '@sentry/types'; import { SentrySpanProcessor } from '../src/spanprocessor'; @@ -220,6 +220,102 @@ describe('SentrySpanProcessor', () => { parentOtelSpan.end(); }); }); + + it('sets status for transaction', async () => { + const otelSpan = provider.getTracer('default').startSpan('GET /users'); + + const transaction = getSpanForOtelSpan(otelSpan) as Transaction; + + // status is only set after end + expect(transaction?.status).toBe(undefined); + + otelSpan.end(); + + expect(transaction?.status).toBe('ok'); + }); + + it('sets status for span', async () => { + const tracer = provider.getTracer('default'); + + tracer.startActiveSpan('GET /users', parentOtelSpan => { + tracer.startActiveSpan('SELECT * FROM users;', child => { + const sentrySpan = getSpanForOtelSpan(child); + + expect(sentrySpan?.status).toBe(undefined); + + child.end(); + + expect(sentrySpan?.status).toBe('ok'); + + parentOtelSpan.end(); + }); + }); + }); + + const statusTestTable: [number, undefined | string, undefined | string, SpanStatusType][] = [ + [-1, undefined, undefined, 'unknown_error'], + [3, undefined, undefined, 'unknown_error'], + [0, undefined, undefined, 'ok'], + [1, undefined, undefined, 'ok'], + [2, undefined, undefined, 'unknown_error'], + + // http codes + [2, '400', undefined, 'failed_precondition'], + [2, '401', undefined, 'unauthenticated'], + [2, '403', undefined, 'permission_denied'], + [2, '404', undefined, 'not_found'], + [2, '409', undefined, 'aborted'], + [2, '429', undefined, 'resource_exhausted'], + [2, '499', undefined, 'cancelled'], + [2, '500', undefined, 'internal_error'], + [2, '501', undefined, 'unimplemented'], + [2, '503', undefined, 'unavailable'], + [2, '504', undefined, 'deadline_exceeded'], + [2, '999', undefined, 'unknown_error'], + + // grpc codes + [2, undefined, '1', 'cancelled'], + [2, undefined, '2', 'unknown_error'], + [2, undefined, '3', 'invalid_argument'], + [2, undefined, '4', 'deadline_exceeded'], + [2, undefined, '5', 'not_found'], + [2, undefined, '6', 'already_exists'], + [2, undefined, '7', 'permission_denied'], + [2, undefined, '8', 'resource_exhausted'], + [2, undefined, '9', 'failed_precondition'], + [2, undefined, '10', 'aborted'], + [2, undefined, '11', 'out_of_range'], + [2, undefined, '12', 'unimplemented'], + [2, undefined, '13', 'internal_error'], + [2, undefined, '14', 'unavailable'], + [2, undefined, '15', 'data_loss'], + [2, undefined, '16', 'unauthenticated'], + [2, undefined, '999', 'unknown_error'], + + // http takes precedence over grpc + [2, '400', '2', 'failed_precondition'], + ]; + + it.each(statusTestTable)( + 'correctly converts otel span status to sentry status with otelStatus=%i, httpCode=%s, grpcCode=%s', + (otelStatus, httpCode, grpcCode, expected) => { + const otelSpan = provider.getTracer('default').startSpan('GET /users'); + const transaction = getSpanForOtelSpan(otelSpan) as Transaction; + + otelSpan.setStatus({ code: otelStatus }); + + if (httpCode) { + otelSpan.setAttribute(SemanticAttributes.HTTP_STATUS_CODE, httpCode); + } + + if (grpcCode) { + otelSpan.setAttribute(SemanticAttributes.RPC_GRPC_STATUS_CODE, grpcCode); + } + + otelSpan.end(); + expect(transaction?.status).toBe(expected); + }, + ); }); // OTEL expects a custom date format From 0a21086888766076db6b2a2b30dabe1b524ec041 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Fri, 28 Oct 2022 13:21:18 +0200 Subject: [PATCH 4/4] feat(otel): Use scope for setting otel transaction context --- .../opentelemetry-node/src/spanprocessor.ts | 19 ++++---- .../test/spanprocessor.test.ts | 48 ++++++++++++------- 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/packages/opentelemetry-node/src/spanprocessor.ts b/packages/opentelemetry-node/src/spanprocessor.ts index 0fd37e1835fa..3891a326359a 100644 --- a/packages/opentelemetry-node/src/spanprocessor.ts +++ b/packages/opentelemetry-node/src/spanprocessor.ts @@ -1,6 +1,6 @@ import { Context } from '@opentelemetry/api'; import { Span as OtelSpan, SpanProcessor as OtelSpanProcessor } from '@opentelemetry/sdk-trace-base'; -import { getCurrentHub, setContext } from '@sentry/core'; +import { getCurrentHub, withScope } from '@sentry/core'; import { Transaction } from '@sentry/tracing'; import { Span as SentrySpan, TransactionContext } from '@sentry/types'; import { logger } from '@sentry/utils'; @@ -77,14 +77,13 @@ export class SentrySpanProcessor implements OtelSpanProcessor { } if (sentrySpan instanceof Transaction) { - updateContextWithOtelData(otelSpan); updateTransactionWithOtelData(sentrySpan, otelSpan); + finishTransactionWithContextFromOtelData(sentrySpan, otelSpan); } else { updateSpanWithOtelData(sentrySpan, otelSpan); + sentrySpan.finish(otelSpan.endTime[0]); } - sentrySpan.finish(otelSpan.endTime[0]); - this._map.delete(otelSpanId); } @@ -116,10 +115,14 @@ function getTraceData(otelSpan: OtelSpan): Partial { return { spanId, traceId, parentSpanId }; } -function updateContextWithOtelData(otelSpan: OtelSpan): void { - setContext('otel', { - attributes: otelSpan.attributes, - resource: otelSpan.resource.attributes, +function finishTransactionWithContextFromOtelData(transaction: Transaction, otelSpan: OtelSpan): void { + withScope(scope => { + scope.setContext('otel', { + attributes: otelSpan.attributes, + resource: otelSpan.resource.attributes, + }); + + transaction.finish(otelSpan.endTime[0]); }); } diff --git a/packages/opentelemetry-node/test/spanprocessor.test.ts b/packages/opentelemetry-node/test/spanprocessor.test.ts index 8a48ece8c428..e371dc453617 100644 --- a/packages/opentelemetry-node/test/spanprocessor.test.ts +++ b/packages/opentelemetry-node/test/spanprocessor.test.ts @@ -43,9 +43,25 @@ describe('SentrySpanProcessor', () => { return spanProcessor._map.get(otelSpan.spanContext().spanId); } - function getContext() { - const scope = hub.getScope() as unknown as Scope & { _contexts: Contexts }; - return scope._contexts; + function getContext(transaction: Transaction) { + const transactionWithContext = transaction as unknown as Transaction & { _contexts: Contexts }; + return transactionWithContext._contexts; + } + + // monkey-patch finish to store the context at finish time + function monkeyPatchTransactionFinish(transaction: Transaction) { + const monkeyPatchedTransaction = transaction as Transaction & { _contexts: Contexts }; + + // eslint-disable-next-line @typescript-eslint/unbound-method + const originalFinish = monkeyPatchedTransaction.finish; + monkeyPatchedTransaction._contexts = {}; + monkeyPatchedTransaction.finish = function (endTimestamp?: number | undefined) { + monkeyPatchedTransaction._contexts = ( + transaction._hub.getScope() as unknown as Scope & { _contexts: Contexts } + )._contexts; + + return originalFinish.apply(monkeyPatchedTransaction, [endTimestamp]); + }; } it('creates a transaction', async () => { @@ -141,12 +157,15 @@ describe('SentrySpanProcessor', () => { it('sets context for transaction', async () => { const otelSpan = provider.getTracer('default').startSpan('GET /users'); + const transaction = getSpanForOtelSpan(otelSpan) as Transaction; + monkeyPatchTransactionFinish(transaction); + // context is only set after end - expect(getContext()).toEqual({}); + expect(getContext(transaction)).toEqual({}); otelSpan.end(); - expect(getContext()).toEqual({ + expect(getContext(transaction)).toEqual({ otel: { attributes: {}, resource: { @@ -158,26 +177,19 @@ describe('SentrySpanProcessor', () => { }, }); - // Start new transaction, context should remain the same + // Start new transaction const otelSpan2 = provider.getTracer('default').startSpan('GET /companies'); - expect(getContext()).toEqual({ - otel: { - attributes: {}, - resource: { - 'service.name': 'test-service', - 'telemetry.sdk.language': 'nodejs', - 'telemetry.sdk.name': 'opentelemetry', - 'telemetry.sdk.version': '1.7.0', - }, - }, - }); + const transaction2 = getSpanForOtelSpan(otelSpan2) as Transaction; + monkeyPatchTransactionFinish(transaction2); + + expect(getContext(transaction2)).toEqual({}); otelSpan2.setAttribute('test-attribute', 'test-value'); otelSpan2.end(); - expect(getContext()).toEqual({ + expect(getContext(transaction2)).toEqual({ otel: { attributes: { 'test-attribute': 'test-value',