feat: big number support for cbor

bigint support in cbor

Author: kuhe

.changeset/real-pens-smile.md

@@ -0,0 +1,5 @@
+---
+"@smithy/core": minor
+---
+
+support BigInt in cbor

packages/core/src/submodules/cbor/SmithyRpcV2CborProtocol.spec.ts

@@ -155,7 +155,7 @@ describe(SmithyRpcV2CborProtocol.name, () => {
           {
             name: "dummy",
             input: testCase.schema,
-            output: void 0,
+            output: "unit",
             traits: {},
           },
           testCase.input,
@@ -257,7 +257,7 @@ describe(SmithyRpcV2CborProtocol.name, () => {
         const output = await protocol.deserializeResponse(
           {
             name: "dummy",
-            input: void 0,
+            input: "unit",
             output: testCase.schema,
             traits: {},
           },

packages/core/src/submodules/cbor/byte-printer.ts

@@ -0,0 +1,7 @@
+/**
+ * Prints bytes as binary string with numbers.
+ * @param bytes
+ */
+export function printBytes(bytes: Uint8Array) {
+  return [...bytes].map((n) => ("0".repeat(8) + n.toString(2)).slice(-8) + ` (${n})`);
+}

packages/core/src/submodules/cbor/cbor-decode.ts

@@ -1,3 +1,4 @@
+import { NumericValue } from "@smithy/core/serde";
 import { toUtf8 } from "@smithy/util-utf8";
 
 import {
@@ -119,11 +120,33 @@ export function decode(at: Uint32, to: Uint32): CborValueType {
         _offset = offset;
         return castBigInt(negativeInt);
       } else {
-        const value = decode(at + offset, to);
-        const valueOffset = _offset;
+        /* major === majorTag */
+        if (minor === 2 || minor === 3) {
+          const length = decodeCount(at + offset, to);
+
+          let b = BigInt(0);
+          const start = at + offset + _offset;
+          for (let i = start; i < start + length; ++i) {
+            b = (b << BigInt(8)) | BigInt(payload[i]);
+          }
+
+          _offset = offset + length;
+          return minor === 3 ? -b - BigInt(1) : b;
+        } else if (minor === 4) {
+          const decimalFraction = decode(at + offset, to);
+          const [exponent, mantissa] = decimalFraction;
+          const s = mantissa.toString();
+          const numericString = exponent === 0 ? s : s.slice(0, s.length + exponent) + "." + s.slice(exponent);
+
+          return new NumericValue(numericString, "bigDecimal");
+        } else {
+          const value = decode(at + offset, to);
+          const valueOffset = _offset;
+
+          _offset = offset + valueOffset;
 
-        _offset = offset + valueOffset;
-        return tag({ tag: castBigInt(unsignedInt), value });
+          return tag({ tag: castBigInt(unsignedInt), value });
+        }
       }
     case majorUtf8String:
     case majorMap:

packages/core/src/submodules/cbor/cbor-encode.ts

@@ -1,6 +1,8 @@
+import { NumericValue } from "@smithy/core/serde";
 import { fromUtf8 } from "@smithy/util-utf8";
 
 import {
+  alloc,
   CborMajorType,
   extendedFloat16,
   extendedFloat32,
@@ -19,7 +21,6 @@ import {
   tagSymbol,
   Uint64,
 } from "./cbor-types";
-import { alloc } from "./cbor-types";
 
 const USE_BUFFER = typeof Buffer !== "undefined";
 
@@ -152,10 +153,30 @@ export function encode(_input: any): void {
         data[cursor++] = (major << 5) | extendedFloat32;
         dataView.setUint32(cursor, n);
         cursor += 4;
-      } else {
+      } else if (value < BigInt("18446744073709551616")) {
         data[cursor++] = (major << 5) | extendedFloat64;
         dataView.setBigUint64(cursor, value);
         cursor += 8;
+      } else {
+        // refer to https://www.rfc-editor.org/rfc/rfc8949.html#name-bignums
+        const binaryBigInt = value.toString(2);
+        const bigIntBytes = new Uint8Array(Math.ceil(binaryBigInt.length / 8));
+        let b = value;
+        let i = 0;
+        while (bigIntBytes.byteLength - ++i >= 0) {
+          bigIntBytes[bigIntBytes.byteLength - i] = Number(b & BigInt(255));
+          b >>= BigInt(8);
+        }
+        ensureSpace(bigIntBytes.byteLength * 2);
+        data[cursor++] = nonNegative ? 0b110_00010 : 0b110_00011;
+
+        if (USE_BUFFER) {
+          encodeHeader(majorUnstructuredByteString, Buffer.byteLength(bigIntBytes));
+        } else {
+          encodeHeader(majorUnstructuredByteString, bigIntBytes.byteLength);
+        }
+        data.set(bigIntBytes, cursor);
+        cursor += bigIntBytes.byteLength;
       }
       continue;
     } else if (input === null) {
@@ -181,6 +202,18 @@ export function encode(_input: any): void {
       cursor += input.byteLength;
       continue;
     } else if (typeof input === "object") {
+      if (input instanceof NumericValue) {
+        const decimalIndex = input.string.indexOf(".");
+        const exponent = decimalIndex === -1 ? 0 : decimalIndex - input.string.length + 1;
+        const mantissa = BigInt(input.string.replace(".", ""));
+
+        data[cursor++] = 0b110_00100; // major 6, tag 4.
+
+        encodeStack.push(mantissa);
+        encodeStack.push(exponent);
+        encodeHeader(majorList, 2);
+        continue;
+      }
       if (input[tagSymbol]) {
         if ("tag" in input && "value" in input) {
           encodeStack.push(input.value);

packages/core/src/submodules/cbor/cbor.spec.ts

@@ -1,3 +1,4 @@
+import { NumericValue } from "@smithy/core/serde";
 import * as fs from "fs";
 // @ts-ignore
 import JSONbig from "json-bigint";
@@ -88,12 +89,12 @@ describe("cbor", () => {
     {
       name: "negative float",
       data: -3015135.135135135,
-      cbor: allocByteArray([0b111_11011, +193, +71, +0, +239, +145, +76, +27, +173]),
+      cbor: allocByteArray([0b111_11011, 193, 71, 0, 239, 145, 76, 27, 173]),
     },
     {
       name: "positive float",
       data: 3015135.135135135,
-      cbor: allocByteArray([0b111_11011, +65, +71, +0, +239, +145, +76, +27, +173]),
+      cbor: allocByteArray([0b111_11011, 65, 71, 0, 239, 145, 76, 27, 173]),
     },
     {
       name: "various numbers",
@@ -214,6 +215,18 @@ describe("cbor", () => {
         65, 109, 110, 101, 115, 116, 101, 100, 32, 105, 116, 101, 109, 32, 66,
       ]),
     },
+    {
+      name: "object containing big numbers",
+      data: {
+        map: {
+          items: [BigInt(1e80)],
+        },
+      },
+      cbor: allocByteArray([
+        161, 99, 109, 97, 112, 161, 101, 105, 116, 101, 109, 115, 129, 194, 88, 34, 3, 95, 157, 234, 62, 31, 107, 224,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      ]),
+    },
   ];
 
   const toBytes = (hex: string) => {
@@ -226,6 +239,72 @@ describe("cbor", () => {
   };
 
   describe("locally curated scenarios", () => {
+    it("should round-trip bigInteger to major 6 with tag 2", () => {
+      const bigInt = BigInt("1267650600228229401496703205376");
+      const serialized = cbor.serialize(bigInt);
+
+      const major = serialized[0] >> 5;
+      expect(major).toEqual(0b110); // 6
+
+      const tag = serialized[0] & 0b11111;
+      expect(tag).toEqual(0b010); // 2
+
+      const byteStringCount = serialized[1];
+      expect(byteStringCount).toEqual(0b010_01101); // major 2, 13 bytes
+
+      const byteString = serialized.slice(2);
+      expect(byteString).toEqual(allocByteArray([0b000_10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]));
+
+      const deserialized = cbor.deserialize(serialized);
+      expect(deserialized).toEqual(bigInt);
+    });
+
+    it("should round-trip negative bigInteger to major 6 with tag 3", () => {
+      const bigInt = BigInt("-1267650600228229401496703205377");
+      const serialized = cbor.serialize(bigInt);
+
+      const major = serialized[0] >> 5;
+      expect(major).toEqual(0b110); // 6
+
+      const tag = serialized[0] & 0b11111;
+      expect(tag).toEqual(0b011); // 3
+
+      const byteStringCount = serialized[1];
+      expect(byteStringCount).toEqual(0b010_01101); // major 2, 13 bytes
+
+      const byteString = serialized.slice(2);
+      expect(byteString).toEqual(allocByteArray([0b000_10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]));
+
+      const deserialized = cbor.deserialize(serialized);
+      expect(deserialized).toEqual(bigInt);
+    });
+
+    it("should round-trip NumericValue to major 6 with tag 4", () => {
+      for (const bigDecimal of [
+        "10000000000000000000000054.321",
+        "1000000000000000000000000000000000054.134134321",
+        "100000000000000000000000000000000000054.0000000000000001",
+        "100000000000000000000000000000000000054.00510351095130000",
+        "-10000000000000000000000054.321",
+        "-1000000000000000000000000000000000054.134134321",
+        "-100000000000000000000000000000000000054.0000000000000001",
+        "-100000000000000000000000000000000000054.00510351095130000",
+      ]) {
+        const nv = new NumericValue(bigDecimal, "bigDecimal");
+        const serialized = cbor.serialize(nv);
+
+        const major = serialized[0] >> 5;
+        expect(major).toEqual(0b110); // 6
+
+        const tag = serialized[0] & 0b11111;
+        expect(tag).toEqual(0b0100); // 4
+
+        const deserialized = cbor.deserialize(serialized);
+        expect(deserialized).toEqual(nv);
+        expect(deserialized.string).toEqual(nv.string);
+      }
+    });
+
     it("should throw an error if serializing a tag with missing properties", () => {
       expect(() =>
         cbor.serialize({

packages/core/src/submodules/protocols/HttpBindingProtocol.spec.ts

@@ -5,6 +5,7 @@ import {
   CodecSettings,
   HandlerExecutionContext,
   HttpResponse as IHttpResponse,
+  MetadataBearer,
   OperationSchema,
   ResponseMetadata,
   ShapeDeserializer,
@@ -64,7 +65,7 @@ describe(HttpBindingProtocol.name, () => {
     });
 
     const protocol = new StringRestProtocol();
-    const output = await protocol.deserializeResponse(
+    const output = (await protocol.deserializeResponse(
       op(
         "",
         "",
@@ -87,7 +88,7 @@ describe(HttpBindingProtocol.name, () => {
       ),
       {} as any,
       response
-    );
+    )) as Partial<MetadataBearer>;
     delete output.$metadata;
     expect(output).toEqual({
       timestampList: [new Date("2019-12-16T23:48:18.000Z"), new Date("2019-12-16T23:48:18.000Z")],
@@ -104,7 +105,7 @@ describe(HttpBindingProtocol.name, () => {
     });
 
     const protocol = new StringRestProtocol();
-    const output = await protocol.deserializeResponse(
+    const output = (await protocol.deserializeResponse(
       op(
         "",
         "",
@@ -127,7 +128,7 @@ describe(HttpBindingProtocol.name, () => {
       ),
       {} as any,
       response
-    );
+    )) as Partial<MetadataBearer>;
     delete output.$metadata;
     expect(output).toEqual({
       httpPrefixHeaders: {

packages/core/src/submodules/serde/parse-utils.spec.ts

@@ -22,6 +22,8 @@ import {
 } from "./parse-utils";
 import { expectBoolean, expectNumber, expectString } from "./parse-utils";
 
+logger.warn = () => {};
+
 describe("parseBoolean", () => {
   it('Returns true for "true"', () => {
     expect(parseBoolean("true")).toEqual(true);

packages/core/src/submodules/serde/value/NumericValue.spec.ts

@@ -9,4 +9,11 @@ describe(NumericValue.name, () => {
     expect(num.string).toEqual("1.0");
     expect(num.type).toEqual("bigDecimal");
   });
+
+  it("allows only numeric digits and at most one decimal point", () => {
+    expect(() => nv("a")).toThrow();
+    expect(() => nv("1.0.1")).toThrow();
+    expect(() => nv("-10.1")).not.toThrow();
+    expect(() => nv("-.101")).not.toThrow();
+  });
 });

packages/core/src/submodules/serde/value/NumericValue.ts

@@ -24,7 +24,41 @@ export class NumericValue {
   public constructor(
     public readonly string: string,
     public readonly type: NumericType
-  ) {}
+  ) {
+    let dot = 0;
+    for (let i = 0; i < string.length; ++i) {
+      const char = string.charCodeAt(i);
+      if (i === 0 && char === 45) {
+        // negation prefix "-"
+        continue;
+      }
+      if (char === 46) {
+        // decimal point "."
+        if (dot) {
+          throw new Error("@smithy/core/serde - NumericValue must contain at most one decimal point.");
+        }
+        dot = 1;
+        continue;
+      }
+      if (char < 48 || char > 57) {
+        // not in 0 through 9
+        throw new Error(
+          `@smithy/core/serde - NumericValue must only contain [0-9], at most one decimal point ".", and an optional negation prefix "-".`
+        );
+      }
+    }
+  }
+
+  public [Symbol.hasInstance](object: unknown) {
+    if (!object || typeof object !== "object") {
+      return false;
+    }
+    const _nv = object as NumericValue;
+    if (typeof _nv.string === "string" && typeof _nv.type === "string" && _nv.constructor?.name === "NumericValue") {
+      return true;
+    }
+    return false;
+  }
 }
 
 /**

packages/core/vitest.config.ts

@@ -5,5 +5,6 @@ export default defineConfig({
     exclude: ["**/*.{integ,e2e,browser}.spec.ts"],
     include: ["**/*.spec.ts"],
     environment: "node",
+    hideSkippedTests: true,
   },
 });