diff --git a/.changeset/ten-taxis-deliver.md b/.changeset/ten-taxis-deliver.md new file mode 100644 index 00000000000..8981b51f840 --- /dev/null +++ b/.changeset/ten-taxis-deliver.md @@ -0,0 +1,6 @@ +--- +"@smithy/types": patch +"@smithy/core": patch +--- + +schema serde: http binding and cbor serializer refactoring diff --git a/packages/core/src/submodules/cbor/CborCodec.ts b/packages/core/src/submodules/cbor/CborCodec.ts index 4624a4520f8..263a8613423 100644 --- a/packages/core/src/submodules/cbor/CborCodec.ts +++ b/packages/core/src/submodules/cbor/CborCodec.ts @@ -1,6 +1,6 @@ import { NormalizedSchema } from "@smithy/core/schema"; -import { copyDocumentWithTransform, parseEpochTimestamp } from "@smithy/core/serde"; -import { Codec, Schema, SchemaRef, SerdeFunctions, ShapeDeserializer, ShapeSerializer } from "@smithy/types"; +import { parseEpochTimestamp } from "@smithy/core/serde"; +import { Codec, Schema, SerdeFunctions, ShapeDeserializer, ShapeSerializer } from "@smithy/types"; import { cbor } from "./cbor"; import { dateToTag } from "./parseCborBody"; @@ -40,38 +40,80 @@ export class CborShapeSerializer implements ShapeSerializer { } public write(schema: Schema, value: unknown): void { - this.value = copyDocumentWithTransform(value, schema, (_: any, schemaRef: SchemaRef) => { - if (_ instanceof Date) { - return dateToTag(_); - } - if (_ instanceof Uint8Array) { - return _; - } + this.value = this.serialize(schema, value); + } - const ns = NormalizedSchema.of(schemaRef); - const sparse = !!ns.getMergedTraits().sparse; + /** + * Recursive serializer transform that copies and prepares the user input object + * for CBOR serialization. + */ + public serialize(schema: Schema, source: unknown): any { + const ns = NormalizedSchema.of(schema); - if (ns.isListSchema() && Array.isArray(_)) { - if (!sparse) { - return _.filter((item) => item != null); + switch (typeof source) { + case "undefined": + return null; + case "boolean": + case "number": + case "string": + case "bigint": + case "symbol": + return source; + case "function": + case "object": + if (source === null) { + return null; } - } else if (_ && typeof _ === "object") { - const members = ns.getMemberSchemas(); - const isStruct = ns.isStructSchema(); - if (!sparse || isStruct) { - for (const [k, v] of Object.entries(_)) { - const filteredOutByNonSparse = !sparse && v == null; - const filteredOutByUnrecognizedMember = isStruct && !(k in members); - if (filteredOutByNonSparse || filteredOutByUnrecognizedMember) { - delete _[k]; + + const sourceObject = source as Record; + const sparse = !!ns.getMergedTraits().sparse; + + if (ns.isListSchema() && Array.isArray(sourceObject)) { + const newArray = []; + let i = 0; + for (const item of sourceObject) { + const value = this.serialize(ns.getValueSchema(), item); + if (value != null || sparse) { + newArray[i++] = value; } } - return _; + return newArray; } - } - - return _; - }); + if (sourceObject instanceof Uint8Array) { + const newBytes = new Uint8Array(sourceObject.byteLength); + newBytes.set(sourceObject, 0); + return newBytes; + } + if (sourceObject instanceof Date) { + return dateToTag(sourceObject); + } + const newObject = {} as any; + if (ns.isMapSchema()) { + for (const key of Object.keys(sourceObject)) { + const value = this.serialize(ns.getValueSchema(), sourceObject[key]); + if (value != null || sparse) { + newObject[key] = value; + } + } + } else if (ns.isStructSchema()) { + for (const [key, memberSchema] of ns.structIterator()) { + const value = this.serialize(memberSchema, sourceObject[key]); + if (value != null) { + newObject[key] = value; + } + } + } else if (ns.isDocumentSchema()) { + for (const key of Object.keys(sourceObject)) { + const value = this.serialize(ns.getValueSchema(), sourceObject[key]); + if (value != null) { + newObject[key] = value; + } + } + } + return newObject; + default: + return source; + } } public flush(): Uint8Array { diff --git a/packages/core/src/submodules/cbor/SmithyRpcV2CborProtocol.spec.ts b/packages/core/src/submodules/cbor/SmithyRpcV2CborProtocol.spec.ts index 4b4271a79ca..315ee57ee03 100644 --- a/packages/core/src/submodules/cbor/SmithyRpcV2CborProtocol.spec.ts +++ b/packages/core/src/submodules/cbor/SmithyRpcV2CborProtocol.spec.ts @@ -103,9 +103,9 @@ describe(SmithyRpcV2CborProtocol.name, () => { 0, ["mySparseList", "myRegularList", "mySparseMap", "myRegularMap"], [ - [() => list("", "MyList", { sparse: 1 }, SCHEMA.NUMERIC), {}], + [() => list("", "MySparseList", { sparse: 1 }, SCHEMA.NUMERIC), {}], [() => list("", "MyList", {}, SCHEMA.NUMERIC), {}], - [() => map("", "MyMap", { sparse: 1 }, SCHEMA.STRING, SCHEMA.NUMERIC), {}], + [() => map("", "MySparseMap", { sparse: 1 }, SCHEMA.STRING, SCHEMA.NUMERIC), {}], [() => map("", "MyMap", {}, SCHEMA.STRING, SCHEMA.NUMERIC), {}], ] ), diff --git a/packages/core/src/submodules/protocols/HttpBindingProtocol.spec.ts b/packages/core/src/submodules/protocols/HttpBindingProtocol.spec.ts index da32f45f262..e358d581892 100644 --- a/packages/core/src/submodules/protocols/HttpBindingProtocol.spec.ts +++ b/packages/core/src/submodules/protocols/HttpBindingProtocol.spec.ts @@ -1,4 +1,4 @@ -import { op, SCHEMA, struct } from "@smithy/core/schema"; +import { map, op, SCHEMA, struct } from "@smithy/core/schema"; import { HttpResponse } from "@smithy/protocol-http"; import { Codec, @@ -8,6 +8,8 @@ import { MetadataBearer, OperationSchema, ResponseMetadata, + Schema, + SerdeFunctions, ShapeDeserializer, ShapeSerializer, } from "@smithy/types"; @@ -41,9 +43,11 @@ describe(HttpBindingProtocol.name, () => { public getShapeId(): string { throw new Error("Method not implemented."); } + public getPayloadCodec(): Codec { throw new Error("Method not implemented."); } + protected handleError( operationSchema: OperationSchema, context: HandlerExecutionContext, @@ -157,4 +161,53 @@ describe(HttpBindingProtocol.name, () => { ); expect(request.path).toEqual("/custom/Operation"); }); + + it("can deserialize a prefix header binding and header binding from the same header", async () => { + type TestSignature = ( + schema: Schema, + context: HandlerExecutionContext & SerdeFunctions, + response: IHttpResponse, + dataObject: any + ) => Promise; + const deserializeHttpMessage = ((StringRestProtocol.prototype as any).deserializeHttpMessage as TestSignature).bind( + { + deserializer: new FromStringShapeDeserializer({ + httpBindings: true, + timestampFormat: { + useTrait: true, + default: SCHEMA.TIMESTAMP_EPOCH_SECONDS, + }, + }), + } + ); + const httpResponse: IHttpResponse = { + statusCode: 200, + headers: { + "my-header": "header-value", + }, + }; + + const dataObject = {}; + await deserializeHttpMessage( + struct( + "", + "Struct", + 0, + ["prefixHeaders", "header"], + [ + [map("", "Map", 0, 0, 0), { httpPrefixHeaders: "my-" }], + [0, { httpHeader: "my-header" }], + ] + ), + {} as any, + httpResponse, + dataObject + ); + expect(dataObject).toEqual({ + prefixHeaders: { + header: "header-value", + }, + header: "header-value", + }); + }); }); diff --git a/packages/core/src/submodules/protocols/HttpBindingProtocol.ts b/packages/core/src/submodules/protocols/HttpBindingProtocol.ts index c754d26248b..6bbe3054102 100644 --- a/packages/core/src/submodules/protocols/HttpBindingProtocol.ts +++ b/packages/core/src/submodules/protocols/HttpBindingProtocol.ts @@ -1,15 +1,19 @@ import { NormalizedSchema, SCHEMA } from "@smithy/core/schema"; +import { splitEvery, splitHeader } from "@smithy/core/serde"; import { HttpRequest } from "@smithy/protocol-http"; import { Endpoint, EndpointBearer, + EventStreamSerdeContext, HandlerExecutionContext, HttpRequest as IHttpRequest, HttpResponse as IHttpResponse, MetadataBearer, OperationSchema, + Schema, SerdeFunctions, } from "@smithy/types"; +import { sdkStreamMixin } from "@smithy/util-stream"; import { collectBody } from "./collect-stream-body"; import { extendedEncodeURIComponent } from "./extended-encode-uri-component"; @@ -226,4 +230,127 @@ export abstract class HttpBindingProtocol extends HttpProtocol { return output; } + + /** + * The base method ignores HTTP bindings. + * + * @deprecated (only this signature) use signature without headerBindings. + * @override + */ + protected async deserializeHttpMessage( + schema: Schema, + context: HandlerExecutionContext & SerdeFunctions, + response: IHttpResponse, + headerBindings: Set, + dataObject: any + ): Promise; + protected async deserializeHttpMessage( + schema: Schema, + context: HandlerExecutionContext & SerdeFunctions, + response: IHttpResponse, + dataObject: any + ): Promise; + protected async deserializeHttpMessage( + schema: Schema, + context: HandlerExecutionContext & SerdeFunctions, + response: IHttpResponse, + arg4: unknown, + arg5?: unknown + ): Promise { + let dataObject: any; + if (arg4 instanceof Set) { + dataObject = arg5; + } else { + dataObject = arg4; + } + + const deserializer = this.deserializer; + const ns = NormalizedSchema.of(schema); + const nonHttpBindingMembers = [] as string[]; + + for (const [memberName, memberSchema] of ns.structIterator()) { + const memberTraits = memberSchema.getMemberTraits(); + + if (memberTraits.httpPayload) { + const isStreaming = memberSchema.isStreaming(); + if (isStreaming) { + const isEventStream = memberSchema.isStructSchema(); + if (isEventStream) { + // streaming event stream (union) + const context = this.serdeContext as unknown as EventStreamSerdeContext; + if (!context.eventStreamMarshaller) { + throw new Error("@smithy/core - HttpProtocol: eventStreamMarshaller missing in serdeContext."); + } + const memberSchemas = memberSchema.getMemberSchemas(); + dataObject[memberName] = context.eventStreamMarshaller.deserialize(response.body, async (event) => { + const unionMember = + Object.keys(event).find((key) => { + return key !== "__type"; + }) ?? ""; + if (unionMember in memberSchemas) { + const eventStreamSchema = memberSchemas[unionMember]; + return { + [unionMember]: await deserializer.read(eventStreamSchema, event[unionMember].body), + }; + } else { + // todo(schema): This union convention is ignored by the event stream marshaller. + // todo(schema): This should be returned to the user instead. + // see "if (deserialized.$unknown) return;" in getUnmarshalledStream.ts + return { + $unknown: event, + }; + } + }); + } else { + // streaming blob body + dataObject[memberName] = sdkStreamMixin(response.body); + } + } else if (response.body) { + const bytes: Uint8Array = await collectBody(response.body, context as SerdeFunctions); + if (bytes.byteLength > 0) { + dataObject[memberName] = await deserializer.read(memberSchema, bytes); + } + } + } else if (memberTraits.httpHeader) { + const key = String(memberTraits.httpHeader).toLowerCase(); + const value = response.headers[key]; + if (null != value) { + if (memberSchema.isListSchema()) { + const headerListValueSchema = memberSchema.getValueSchema(); + let sections: string[]; + if ( + headerListValueSchema.isTimestampSchema() && + headerListValueSchema.getSchema() === SCHEMA.TIMESTAMP_DEFAULT + ) { + sections = splitEvery(value, ",", 2); + } else { + sections = splitHeader(value); + } + const list = []; + for (const section of sections) { + list.push(await deserializer.read([headerListValueSchema, { httpHeader: key }], section.trim())); + } + dataObject[memberName] = list; + } else { + dataObject[memberName] = await deserializer.read(memberSchema, value); + } + } + } else if (memberTraits.httpPrefixHeaders !== undefined) { + dataObject[memberName] = {}; + for (const [header, value] of Object.entries(response.headers)) { + if (header.startsWith(memberTraits.httpPrefixHeaders)) { + dataObject[memberName][header.slice(memberTraits.httpPrefixHeaders.length)] = await deserializer.read( + [memberSchema.getValueSchema(), { httpHeader: header }], + value + ); + } + } + } else if (memberTraits.httpResponseCode) { + dataObject[memberName] = response.statusCode; + } else { + nonHttpBindingMembers.push(memberName); + } + } + return nonHttpBindingMembers; + } } diff --git a/packages/core/src/submodules/protocols/HttpProtocol.spec.ts b/packages/core/src/submodules/protocols/HttpProtocol.spec.ts index d2d183a23a0..51f831239fd 100644 --- a/packages/core/src/submodules/protocols/HttpProtocol.spec.ts +++ b/packages/core/src/submodules/protocols/HttpProtocol.spec.ts @@ -6,7 +6,7 @@ import { HttpProtocol } from "./HttpProtocol"; import { FromStringShapeDeserializer } from "./serde/FromStringShapeDeserializer"; describe(HttpProtocol.name, () => { - it("can deserialize a prefix header binding and header binding from the same header", async () => { + it("ignores http bindings (only HttpBindingProtocol uses them)", async () => { type TestSignature = ( schema: Schema, context: HandlerExecutionContext & SerdeFunctions, @@ -46,10 +46,7 @@ describe(HttpProtocol.name, () => { dataObject ); expect(dataObject).toEqual({ - prefixHeaders: { - header: "header-value", - }, - header: "header-value", + // headers were ignored }); }); }); diff --git a/packages/core/src/submodules/protocols/HttpProtocol.ts b/packages/core/src/submodules/protocols/HttpProtocol.ts index 10affc03bbd..9cd6272945c 100644 --- a/packages/core/src/submodules/protocols/HttpProtocol.ts +++ b/packages/core/src/submodules/protocols/HttpProtocol.ts @@ -1,5 +1,4 @@ -import { NormalizedSchema, SCHEMA } from "@smithy/core/schema"; -import { splitEvery, splitHeader } from "@smithy/core/serde"; +import { NormalizedSchema } from "@smithy/core/schema"; import { HttpRequest, HttpResponse } from "@smithy/protocol-http"; import { ClientProtocol, @@ -7,7 +6,6 @@ import { Endpoint, EndpointBearer, EndpointV2, - EventStreamSerdeContext, HandlerExecutionContext, HttpRequest as IHttpRequest, HttpResponse as IHttpResponse, @@ -19,9 +17,6 @@ import { ShapeDeserializer, ShapeSerializer, } from "@smithy/types"; -import { sdkStreamMixin } from "@smithy/util-stream"; - -import { collectBody } from "./collect-stream-body"; /** * Abstract base for HTTP-based client protocols. @@ -144,7 +139,9 @@ export abstract class HttpProtocol implements ClientProtocol { - let dataObject: any; - if (arg4 instanceof Set) { - dataObject = arg5; - } else { - dataObject = arg4; - } - - const deserializer = this.deserializer; - const ns = NormalizedSchema.of(schema); - const nonHttpBindingMembers = [] as string[]; - - for (const [memberName, memberSchema] of ns.structIterator()) { - const memberTraits = memberSchema.getMemberTraits(); - - if (memberTraits.httpPayload) { - const isStreaming = memberSchema.isStreaming(); - if (isStreaming) { - const isEventStream = memberSchema.isStructSchema(); - if (isEventStream) { - // streaming event stream (union) - const context = this.serdeContext as unknown as EventStreamSerdeContext; - if (!context.eventStreamMarshaller) { - throw new Error("@smithy/core - HttpProtocol: eventStreamMarshaller missing in serdeContext."); - } - const memberSchemas = memberSchema.getMemberSchemas(); - dataObject[memberName] = context.eventStreamMarshaller.deserialize(response.body, async (event) => { - const unionMember = - Object.keys(event).find((key) => { - return key !== "__type"; - }) ?? ""; - if (unionMember in memberSchemas) { - const eventStreamSchema = memberSchemas[unionMember]; - return { - [unionMember]: await deserializer.read(eventStreamSchema, event[unionMember].body), - }; - } else { - // this union convention is ignored by the event stream marshaller. - return { - $unknown: event, - }; - } - }); - } else { - // streaming blob body - dataObject[memberName] = sdkStreamMixin(response.body); - } - } else if (response.body) { - const bytes: Uint8Array = await collectBody(response.body, context as SerdeFunctions); - if (bytes.byteLength > 0) { - dataObject[memberName] = await deserializer.read(memberSchema, bytes); - } - } - } else if (memberTraits.httpHeader) { - const key = String(memberTraits.httpHeader).toLowerCase(); - const value = response.headers[key]; - if (null != value) { - if (memberSchema.isListSchema()) { - const headerListValueSchema = memberSchema.getValueSchema(); - let sections: string[]; - if ( - headerListValueSchema.isTimestampSchema() && - headerListValueSchema.getSchema() === SCHEMA.TIMESTAMP_DEFAULT - ) { - sections = splitEvery(value, ",", 2); - } else { - sections = splitHeader(value); - } - const list = []; - for (const section of sections) { - list.push(await deserializer.read([headerListValueSchema, { httpHeader: key }], section.trim())); - } - dataObject[memberName] = list; - } else { - dataObject[memberName] = await deserializer.read(memberSchema, value); - } - } - } else if (memberTraits.httpPrefixHeaders !== undefined) { - dataObject[memberName] = {}; - for (const [header, value] of Object.entries(response.headers)) { - if (header.startsWith(memberTraits.httpPrefixHeaders)) { - dataObject[memberName][header.slice(memberTraits.httpPrefixHeaders.length)] = await deserializer.read( - [memberSchema.getValueSchema(), { httpHeader: header }], - value - ); - } - } - } else if (memberTraits.httpResponseCode) { - dataObject[memberName] = response.statusCode; - } else { - nonHttpBindingMembers.push(memberName); - } - } - return nonHttpBindingMembers; + void schema; + void context; + void response; + void arg4; + void arg5; + // This method is preserved for backwards compatibility. + // It should remain unused. + return []; } } diff --git a/packages/core/src/submodules/serde/copyDocumentWithTransform.ts b/packages/core/src/submodules/serde/copyDocumentWithTransform.ts index f5146a231d8..fbaedb2876e 100644 --- a/packages/core/src/submodules/serde/copyDocumentWithTransform.ts +++ b/packages/core/src/submodules/serde/copyDocumentWithTransform.ts @@ -1,61 +1,11 @@ -import { NormalizedSchema } from "@smithy/core/schema"; import { SchemaRef } from "@smithy/types"; /** * @internal + * @deprecated the former functionality has been internalized to the CborCodec. */ export const copyDocumentWithTransform = ( source: any, schemaRef: SchemaRef, transform: (_: any, schemaRef: SchemaRef) => any = (_) => _ -): any => { - const ns = NormalizedSchema.of(schemaRef); - switch (typeof source) { - case "undefined": - case "boolean": - case "number": - case "string": - case "bigint": - case "symbol": - return transform(source, ns); - case "function": - case "object": - if (source === null) { - return transform(null, ns); - } - if (Array.isArray(source)) { - const newArray = new Array(source.length); - let i = 0; - for (const item of source) { - newArray[i++] = copyDocumentWithTransform(item, ns.getValueSchema(), transform); - } - return transform(newArray, ns); - } - if ("byteLength" in (source as Uint8Array)) { - const newBytes = new Uint8Array(source.byteLength); - newBytes.set(source, 0); - return transform(newBytes, ns); - } - if (source instanceof Date) { - return transform(source, ns); - } - const newObject = {} as any; - if (ns.isMapSchema()) { - for (const key of Object.keys(source)) { - newObject[key] = copyDocumentWithTransform(source[key], ns.getValueSchema(), transform); - } - } else if (ns.isStructSchema()) { - for (const [key, memberSchema] of ns.structIterator()) { - newObject[key] = copyDocumentWithTransform(source[key], memberSchema, transform); - } - } else if (ns.isDocumentSchema()) { - for (const key of Object.keys(source)) { - newObject[key] = copyDocumentWithTransform(source[key], ns.getValueSchema(), transform); - } - } - - return transform(newObject, ns); - default: - return transform(source, ns); - } -}; +): any => source; diff --git a/packages/types/src/schema/schema.ts b/packages/types/src/schema/schema.ts index a75d35bff3c..cbcebad4dd2 100644 --- a/packages/types/src/schema/schema.ts +++ b/packages/types/src/schema/schema.ts @@ -142,6 +142,7 @@ export type SchemaTraitsObject = { mediaType?: string; error?: "client" | "server"; + streaming?: 1; [traitName: string]: unknown; }; diff --git a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/schema/SchemaTraitFilterIndex.java b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/schema/SchemaTraitFilterIndex.java index 2df0368af2e..41cfe9e8e49 100644 --- a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/schema/SchemaTraitFilterIndex.java +++ b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/schema/SchemaTraitFilterIndex.java @@ -60,21 +60,33 @@ final class SchemaTraitFilterIndex implements KnowledgeIndex { private final Set includedTraits = new HashSet<>( // (wrapped for mutability) SetUtils.of( - SparseTrait.ID, + SparseTrait.ID, // Shape serde + // todo(schema) needs schema logger implementation SensitiveTrait.ID, + // todo(schema) needs automatic generation by protocol serializer IdempotencyTokenTrait.ID, - JsonNameTrait.ID, - MediaTypeTrait.ID, - XmlAttributeTrait.ID, - XmlFlattenedTrait.ID, - XmlNameTrait.ID, - XmlNamespaceTrait.ID, + JsonNameTrait.ID, // Shape serde + MediaTypeTrait.ID, // JSON shape serde + XmlAttributeTrait.ID, // XML shape serde + XmlFlattenedTrait.ID, // XML shape serde + XmlNameTrait.ID, // XML shape serde + XmlNamespaceTrait.ID, // XML shape serde + StreamingTrait.ID, // HttpBindingProtocol handles streaming + payload members. + EndpointTrait.ID, // HttpProtocol + ErrorTrait.ID, // set by the ServiceException runtime classes. + RequiresLengthTrait.ID, // unhandled + + // todo(schema) EventHeaderTrait.ID, + // todo(schema) EventPayloadTrait.ID, - StreamingTrait.ID, - RequiresLengthTrait.ID, - EndpointTrait.ID, + + // afaict, HttpErrorTrait is ignored by the client. The discriminator selects the error structure + // but the actual HTTP response status code is used with no particular comparison + // with the trait's error code. HttpErrorTrait.ID, + // the following HTTP traits are handled by HTTP binding protocol base class. + HttpTrait.ID, HttpHeaderTrait.ID, HttpQueryTrait.ID, HttpLabelTrait.ID, @@ -82,9 +94,7 @@ final class SchemaTraitFilterIndex implements KnowledgeIndex { HttpPrefixHeadersTrait.ID, HttpQueryParamsTrait.ID, HttpResponseCodeTrait.ID, - HostLabelTrait.ID, - ErrorTrait.ID, - HttpTrait.ID + HostLabelTrait.ID ) ); private final Map cache = new HashMap<>();