rework schemas

Author: kuhe

packages/core/src/submodules/cbor/CborCodec.ts

@@ -1,15 +1,6 @@
-import { deref, ListSchema, MapSchema, StructureSchema } from "@smithy/core/schema";
+import { NormalizedSchema } from "@smithy/core/schema";
 import { copyDocumentWithTransform, parseEpochTimestamp } from "@smithy/core/serde";
-import {
-  Codec,
-  MemberSchema,
-  Schema,
-  SchemaRef,
-  SerdeContext,
-  ShapeDeserializer,
-  ShapeSerializer,
-  TraitsSchema,
-} from "@smithy/types";
+import { Codec, Schema, SchemaRef, SerdeContext, ShapeDeserializer, ShapeSerializer } from "@smithy/types";
 
 import { cbor } from "./cbor";
 import { dateToTag } from "./parseCborBody";
@@ -46,17 +37,19 @@ export class CborShapeSerializer implements ShapeSerializer<Uint8Array> {
       if (_ instanceof Date) {
         return dateToTag(_);
       }
-      const schema = deref((schemaRef as MemberSchema)?.[0] ?? schemaRef);
       if (_ instanceof Uint8Array) {
         return _;
       }
-      const sparse = (schema as TraitsSchema)?.traits?.sparse;
+
+      const ns = NormalizedSchema.of(schemaRef);
+      const sparse = !!ns.getMergedTraits().sparse;
+
       if (Array.isArray(_)) {
         if (!sparse) {
           return _.filter((item) => item != null);
         }
       } else if (_ && typeof _ === "object") {
-        if (!sparse) {
+        if (!sparse || ns.isStructSchema()) {
           for (const [k, v] of Object.entries(_)) {
             if (v == null) {
               delete _[k];
@@ -65,6 +58,7 @@ export class CborShapeSerializer implements ShapeSerializer<Uint8Array> {
           return _;
         }
       }
+
       return _;
     });
   }
@@ -88,15 +82,20 @@ export class CborShapeDeserializer implements ShapeDeserializer {
     return this.readValue(schema, data);
   }
 
-  private readValue(schema: Schema, value: any): any {
-    if (typeof schema === "string") {
-      if (schema === "time" || schema === "epoch-seconds" || schema === "date-time") {
+  private readValue(_schema: Schema, value: any): any {
+    const ns = NormalizedSchema.of(_schema);
+    const schema = ns.getSchema();
+
+    if (typeof schema === "number") {
+      if (ns.isTimestampSchema()) {
+        // format is ignored.
         return parseEpochTimestamp(value);
       }
-      if (schema === "blob" || schema === "streaming-blob") {
+      if (ns.isBlobSchema()) {
         return value;
       }
     }
+
     switch (typeof value) {
       case "undefined":
       case "boolean":
@@ -116,38 +115,36 @@ export class CborShapeDeserializer implements ShapeDeserializer {
         if (value instanceof Date) {
           return value;
         }
-        const traits =
-          Array.isArray(schema) && schema.length >= 2
-            ? {
-                ...(deref((schema as MemberSchema)[0]) as TraitsSchema)?.traits,
-                ...(schema as MemberSchema)[1],
-              }
-            : (deref(schema) as TraitsSchema)?.traits;
 
         if (Array.isArray(value)) {
           const newArray = [];
+          const memberSchema = ns.getValueSchema();
+          const sparse = ns.isListSchema() && !!ns.getMergedTraits().sparse;
+
           for (const item of value) {
-            newArray.push(this.readValue(schema instanceof ListSchema ? deref(schema.valueSchema) : void 0, item));
-            if (!traits?.sparse) {
-              if (newArray[newArray.length - 1] == null) {
-                newArray.pop();
-              }
+            newArray.push(this.readValue(memberSchema, item));
+            if (!sparse && newArray[newArray.length - 1] == null) {
+              newArray.pop();
             }
           }
           return newArray;
         }
 
         const newObject = {} as any;
         for (const key of Object.keys(value)) {
-          const targetSchema =
-            schema instanceof StructureSchema
-              ? deref(schema.members[key]?.[0])
-              : schema instanceof MapSchema
-                ? deref(schema.valueSchema)
-                : void 0;
-          newObject[key] = this.readValue(targetSchema, value[key]);
-          if (!traits?.sparse && newObject[key] == null) {
-            delete newObject[key];
+          if (ns.isMapSchema() || ns.isDocumentSchema()) {
+            const targetSchema = ns.getValueSchema();
+            newObject[key] = this.readValue(targetSchema, value[key]);
+
+            if (ns.isMapSchema() && newObject[key] == null && !ns.getMergedTraits().sparse) {
+              delete newObject[key];
+            }
+          } else if (ns.isStructSchema()) {
+            const targetSchema = ns.getMemberSchema(key);
+            if (targetSchema === undefined) {
+              continue;
+            }
+            newObject[key] = this.readValue(targetSchema, value[key]);
           }
         }
         return newObject;

packages/core/src/submodules/cbor/SmithyRpcV2CborProtocol.spec.ts

@@ -154,10 +154,10 @@ describe(SmithyRpcV2CborProtocol.name, () => {
         const protocol = new SmithyRpcV2CborProtocol();
         const httpRequest = await protocol.serializeRequest(
           {
+            name: "dummy",
             input: testCase.schema,
             output: void 0,
             traits: {},
-            errors: [],
           },
           testCase.input,
           {
@@ -249,10 +249,10 @@ describe(SmithyRpcV2CborProtocol.name, () => {
         const protocol = new SmithyRpcV2CborProtocol();
         const output = await protocol.deserializeResponse(
           {
+            name: "dummy",
             input: void 0,
             output: testCase.schema,
             traits: {},
-            errors: [],
           },
           {},
           new HttpResponse({

packages/core/src/submodules/cbor/SmithyRpcV2CborProtocol.ts

@@ -1,4 +1,4 @@
-import { HttpInterceptingShapeSerializer,HttpProtocol } from "@smithy/core/protocols";
+import { HttpInterceptingShapeSerializer, HttpProtocol } from "@smithy/core/protocols";
 import { deref, ErrorSchema, OperationSchema, TypeRegistry } from "@smithy/core/schema";
 import type {
   HandlerExecutionContext,
@@ -36,6 +36,10 @@ export class SmithyRpcV2CborProtocol extends HttpProtocol {
       delete request.body;
       delete request.headers["content-type"];
     } else {
+      if (!request.body) {
+        this.serializer.write(15, {})
+        request.body = this.serializer.flush();
+      }
       try {
         request.headers["content-length"] = String((request.body as Uint8Array).byteLength);
       } catch (e) {}
@@ -82,6 +86,7 @@ export class SmithyRpcV2CborProtocol extends HttpProtocol {
       // TODO(schema) throw client base exception using the dataObject.
       throw new Error("schema not found for " + error);
     }
+
     const message = dataObject.message ?? dataObject.Message ?? "Unknown";
     const exception = new errorSchema.ctor(message);
     Object.assign(exception, {

packages/core/src/submodules/protocols/HttpProtocol.ts

@@ -87,6 +87,9 @@ export abstract class HttpProtocol implements Protocol<IHttpRequest, IHttpRespon
 
     for (const memberName of Object.keys(_input)) {
       const memberNs = ns.getMemberSchema(memberName);
+      if (memberNs === undefined) {
+        continue;
+      }
       const memberSchema = memberNs.getSchema();
       const memberTraits = memberNs.getMergedTraits();
       const inputMember = (_input as any)[memberName] as any;
@@ -122,13 +125,13 @@ export abstract class HttpProtocol implements Protocol<IHttpRequest, IHttpRespon
       } else if (memberTraits.httpPrefixHeaders) {
         for (const [key, val] of Object.entries(inputMember)) {
           const amalgam = memberTraits.httpPrefixHeaders + key;
-          serializer.write([, { httpHeader: amalgam }], val);
+          serializer.write([0, { httpHeader: amalgam }], val);
           headers[amalgam.toLowerCase()] = String(serializer.flush()).toLowerCase();
         }
         delete _input[memberName];
       } else if (memberTraits.httpQueryParams) {
         for (const [key, val] of Object.entries(inputMember)) {
-          serializer.write([, { httpQuery: key }], val);
+          serializer.write([0, { httpQuery: key }], val);
           query[key] = serializer.flush() as string;
         }
         delete _input[memberName];
@@ -178,6 +181,16 @@ export abstract class HttpProtocol implements Protocol<IHttpRequest, IHttpRespon
     const schema = ns.getSchema() as StructureSchema;
 
     let dataObject: any = {};
+
+    if (response.statusCode >= 300) {
+      const bytes: Uint8Array = await collectBody(response.body, context as SerdeContext);
+      if (bytes.byteLength > 0) {
+        Object.assign(dataObject, await deserializer.read(15, bytes));
+      }
+      await this.handleError(operationSchema, context, response, dataObject, this.deserializeMetadata(response));
+      throw new Error("@smithy/core/protocols - HTTP Protocol error handler failed to throw.");
+    }
+
     let hasNonHttpBindingMember = false;
 
     for (const header in response.headers) {
@@ -188,6 +201,9 @@ export abstract class HttpProtocol implements Protocol<IHttpRequest, IHttpRespon
 
     for (const [memberName] of Object.entries(schema?.members ?? {})) {
       const memberSchema = ns.getMemberSchema(memberName);
+      if (memberSchema === undefined) {
+        continue;
+      }
       const memberSchemas = memberSchema.getMemberSchemas();
       const memberTraits = memberSchema.getMemberTraits();
 
@@ -261,10 +277,6 @@ export abstract class HttpProtocol implements Protocol<IHttpRequest, IHttpRespon
       }
     }
 
-    if (response.statusCode >= 300) {
-      await this.handleError(operationSchema, context, response, dataObject, this.deserializeMetadata(response));
-    }
-
     const output: Output = {
       $metadata: this.deserializeMetadata(response),
       ...dataObject,

packages/core/src/submodules/schema/TypeRegistry.ts

@@ -29,26 +29,13 @@ export class TypeRegistry {
    * @param schema - to be registered.
    */
   public register(shapeId: string, schema: ISchema) {
-    const qualifiedName = this.namespace + "#" + shapeId;
+    const qualifiedName = this.normalizeShapeId(shapeId);
     this.schemas.set(qualifiedName, schema);
     if (typeof schema === "object") {
       TypeRegistry.schemaToRegistry.set(schema, this);
     }
   }
 
-  /**
-   * Used to disambiguate e.g. XML values, which are all strings,
-   * into other simple JavaScript types like boolean and number.
-   *
-   * @param simpleTypes - map of shape id strings to simple type.
-   */
-  public registerSimpleTypes(simpleTypes: Record<string, "boolean" | "number" | "bigint" | "bigdecimal">): void {
-    for (const [name, type] of Object.entries(simpleTypes)) {
-      const normalizedName = name.includes("#") ? name : this.namespace + "#" + name;
-      this.simpleTypes[normalizedName] = type;
-    }
-  }
-
   /**
    * Used to disambiguate e.g. XML values, which are all strings,
    * into other simple JavaScript types like boolean and number.
@@ -57,21 +44,19 @@ export class TypeRegistry {
    * @returns simple type of the shape id in this registry.
    */
   public getSimpleType(shapeId: string): string {
-    if (shapeId.includes("#")) {
-      return this.simpleTypes[shapeId];
-    }
-    return this.simpleTypes[this.namespace + "#" + shapeId] ?? "unknown";
+    return this.simpleTypes[this.normalizeShapeId(shapeId)] ?? "unknown";
   }
 
   /**
    * @param shapeId - query.
    * @returns the schema.
    */
   public getSchema(shapeId: string): ISchema {
-    if (shapeId.includes("#")) {
-      return this.schemas.get(shapeId);
+    const id = this.normalizeShapeId(shapeId);
+    if (!this.schemas.has(id)) {
+      throw new Error(`@smithy/core/schema - schema not found for ${id}`);
     }
-    return this.schemas.get(this.namespace + "#" + shapeId);
+    return this.schemas.get(id)!;
   }
 
   /**
@@ -93,4 +78,11 @@ export class TypeRegistry {
     TypeRegistry.registries.delete(this.namespace);
     this.schemas.clear();
   }
+
+  private normalizeShapeId(shapeId: string) {
+    if (shapeId.includes("#")) {
+      return shapeId;
+    }
+    return this.namespace + "#" + shapeId;
+  }
 }

packages/core/src/submodules/schema/index.ts

@@ -6,5 +6,6 @@ export * from "./schemas/OperationSchema";
 export * from "./schemas/ErrorSchema";
 export * from "./schemas/NormalizedSchema";
 export * from "./schemas/Schema";
+export * from "./schemas/SimpleSchema";
 export * from "./schemas/StructureSchema";
 export * from "./TypeRegistry";

packages/core/src/submodules/schema/schemas/ErrorSchema.ts

@@ -7,23 +7,25 @@ export class ErrorSchema extends StructureSchema {
   public constructor(
     public name: string,
     public traits: SchemaTraits,
-    public members: Record<string, [SchemaRef, SchemaTraits]>,
+    public memberNames: string[],
+    public memberList: SchemaRef[],
     /**
      * Constructor for a modeled service exception class that extends Error.
      */
     public ctor: any
   ) {
-    super(name, traits, members);
+    super(name, traits, memberNames, memberList);
   }
 }
 
 export function error(
   name: string,
   traits: SchemaTraits = {},
-  members: Record<string, [SchemaRef, SchemaTraits]> = {},
+  memberNames: string[],
+  memberList: SchemaRef[],
   ctor: any
 ): ErrorSchema {
-  const schema = new ErrorSchema(name, traits, members, ctor);
+  const schema = new ErrorSchema(name, traits, memberNames, memberList, ctor);
   if (TypeRegistry.active) {
     TypeRegistry.active.register(name, schema);
   }

packages/core/src/submodules/schema/schemas/ListSchema.ts

@@ -13,7 +13,7 @@ export class ListSchema extends Schema implements IListSchema {
   }
 }
 
-export function list(name: string, traits: SchemaTraits = {}, valueSchema: SchemaRef = void 0): ListSchema {
+export function list(name: string, traits: SchemaTraits = {}, valueSchema: SchemaRef): ListSchema {
   const schema = new ListSchema(name, traits, typeof valueSchema === "function" ? valueSchema() : valueSchema);
   if (TypeRegistry.active) {
     TypeRegistry.active.register(name, schema);

packages/core/src/submodules/schema/schemas/MapSchema.ts

@@ -13,7 +13,7 @@ export class MapSchema extends Schema implements IMapSchema {
   }
 }
 
-export function map(name: string, traits: SchemaTraits = {}, valueSchema: SchemaRef = void 0): MapSchema {
+export function map(name: string, traits: SchemaTraits = {}, valueSchema: SchemaRef): MapSchema {
   const schema = new MapSchema(name, traits, typeof valueSchema === "function" ? valueSchema() : valueSchema);
   if (TypeRegistry.active) {
     TypeRegistry.active.register(name, schema);