feat(clients): blob helper tests

packages/types/src/blob/blob-types.ts

@@ -3,6 +3,21 @@ import { RuntimeBlobTypes } from "./runtime-blob-types.node";
 /**
  * @public
  *
- * A set of types that can be used as inputs for the blob type.
+ * A union of types that can be used as inputs for the service model
+ * "blob" type when it represents the request's entire payload or body.
+ *
+ * For example, in Lambda::invoke, the payload is modeled as a blob type
+ * and this union applies to it.
+ * In contrast, in Lambda::createFunction the Zip file option is a blob type,
+ * but is not the (entire) payload and this union does not apply.
+ *
+ * Note: not all types are signable by the standard SignatureV4 signer when
+ * used as the request body. For example, in Node.js a Readable stream
+ * is not signable by the default signer.
+ * They are included in the union because it may work in some cases,
+ * but the expected types are primarily string and Uint8Array.
+ *
+ * Additional details may be found in the internal
+ * function "getPayloadHash" in the SignatureV4 module.
  */
 export type BlobTypes = string | ArrayBuffer | ArrayBufferView | Uint8Array | RuntimeBlobTypes;

packages/util-stream/jest.config.e2e.js

@@ -0,0 +1,4 @@
+module.exports = {
+  preset: "ts-jest",
+  testMatch: ["**/*.e2e.spec.ts"],
+};

packages/util-stream/package.json

@@ -9,7 +9,8 @@
     "build:types": "tsc -p tsconfig.types.json",
     "build:types:downlevel": "downlevel-dts dist-types dist-types/ts3.4",
     "clean": "rimraf ./dist-* && rimraf *.tsbuildinfo",
-    "test": "jest"
+    "test": "jest",
+    "test:e2e": "jest -c jest.config.e2e.js"
   },
   "main": "./dist-cjs/index.js",
   "module": "./dist-es/index.js",
@@ -52,13 +53,11 @@
   ],
   "browser": {
     "./dist-es/getAwsChunkedEncodingStream": "./dist-es/getAwsChunkedEncodingStream.browser",
-    "./dist-es/sdk-stream-mixin": "./dist-es/sdk-stream-mixin.browser",
-    "./dist-es/blob/decode": "./dist-es/blob/decode.browser"
+    "./dist-es/sdk-stream-mixin": "./dist-es/sdk-stream-mixin.browser"
   },
   "react-native": {
     "./dist-es/getAwsChunkedEncodingStream": "./dist-es/getAwsChunkedEncodingStream.browser",
-    "./dist-es/sdk-stream-mixin": "./dist-es/sdk-stream-mixin.browser",
-    "./dist-es/blob/decode": "./dist-es/blob/decode.browser"
+    "./dist-es/sdk-stream-mixin": "./dist-es/sdk-stream-mixin.browser"
   },
   "homepage": "https://github.com/aws/aws-sdk-js-v3/tree/main/packages/util-stream",
   "repository": {

packages/util-stream/src/blob/Uint8ArrayBlobAdapter.spec.ts

@@ -1,8 +1,8 @@
-import { BlobAdapter } from "./BlobAdapter";
+import { Uint8ArrayBlobAdapter } from "./Uint8ArrayBlobAdapter";
 
-describe(BlobAdapter.name, () => {
+describe(Uint8ArrayBlobAdapter.name, () => {
   it("extends Uint8Array", () => {
-    const blobby = new BlobAdapter(5);
+    const blobby = new Uint8ArrayBlobAdapter(5);
 
     blobby[-1] = 8;
     blobby[0] = 8;
@@ -26,6 +26,7 @@ describe(BlobAdapter.name, () => {
   });
 
   it("should transform to string synchronously", () => {
-    throw new Error("NYI");
+    const blob = Uint8ArrayBlobAdapter.fromString("test-123");
+    expect(blob.transformToString()).toEqual("test-123");
   });
 });

packages/util-stream/src/blob/Uint8ArrayBlobAdapter.ts

@@ -1,4 +1,4 @@
-import { transformFromObject, transformFromString, transformToString } from "./transforms";
+import { transformFromString, transformToString } from "./transforms";
 
 /**
  * Adapter for conversions of the native Uint8Array type.
@@ -9,12 +9,10 @@ export class Uint8ArrayBlobAdapter extends Uint8Array {
    * @param source - such as a string or Stream.
    * @returns a new Uint8ArrayBlobAdapter extending Uint8Array.
    */
-  public static from(source: unknown): Uint8ArrayBlobAdapter {
+  public static fromString(source: string, encoding = "utf-8"): Uint8ArrayBlobAdapter {
     switch (typeof source) {
       case "string":
-        return transformFromString(source as string);
-      case "object":
-        return transformFromObject(source as object);
+        return transformFromString(source, encoding);
       default:
         throw new Error(`Unsupported conversion from ${typeof source} to Uint8ArrayBlobAdapter.`);
     }

packages/util-stream/src/blob/transforms.ts

@@ -1,24 +1,24 @@
-import { Readable } from "stream";
+import { fromBase64, toBase64 } from "@aws-sdk/util-base64";
+import { fromUtf8, toUtf8 } from "@aws-sdk/util-utf8";
 
 import { Uint8ArrayBlobAdapter } from "./Uint8ArrayBlobAdapter";
 
 /**
  * @internal
  */
 export function transformToString(payload: Uint8Array, encoding = "utf-8"): string {
-  return Buffer.from(payload).toString(encoding as BufferEncoding);
+  if (encoding === "base64") {
+    return toBase64(payload);
+  }
+  return toUtf8(payload);
 }
 
 /**
  * @internal
  */
-export function transformFromString(str: string): Uint8ArrayBlobAdapter {
-  return new Uint8ArrayBlobAdapter(Buffer.from(str));
-}
-
-/**
- * @internal
- */
-export function transformFromObject(obj: object): Uint8ArrayBlobAdapter {
-  return new Uint8ArrayBlobAdapter(obj as any);
+export function transformFromString(str: string, encoding?: string): Uint8ArrayBlobAdapter {
+  if (encoding === "base64") {
+    return Uint8ArrayBlobAdapter.mutate(fromBase64(str));
+  }
+  return Uint8ArrayBlobAdapter.mutate(fromUtf8(str));
 }

packages/util-stream/test/index.mjs

@@ -0,0 +1,6 @@
+/**
+ * function.zip contains this file.
+ */
+export const handler = async (event) => {
+  return event;
+};

packages/util-stream/test/setup.ts

@@ -0,0 +1,149 @@
+import { AttachedPolicy, CreateRoleResponse, GetRoleResponse, IAM } from "@aws-sdk/client-iam";
+import {
+  GetFunctionConfigurationCommandOutput,
+  Lambda,
+  Runtime,
+  waitUntilFunctionActiveV2,
+  waitUntilFunctionUpdated,
+} from "@aws-sdk/client-lambda";
+import fs from "fs";
+export const FunctionName = "aws-sdk-js-v3-e2e-echo";
+export const Handler = "index.handler";
+const LAMBDA_ROLE_NAME = "aws-sdk-js-v3-e2e-LambdaRole";
+
+export async function setup() {
+  const lambda = new Lambda({
+    region: "us-west-2",
+  });
+  const getFn: null | GetFunctionConfigurationCommandOutput = await lambda
+    .getFunctionConfiguration({
+      FunctionName,
+    })
+    .catch(() => null);
+
+  if (getFn) {
+    return;
+  }
+
+  const iam = new IAM({
+    region: "us-west-2",
+  });
+
+  const roleName = LAMBDA_ROLE_NAME;
+  const role: null | GetRoleResponse | CreateRoleResponse = await iam
+    .getRole({
+      RoleName: roleName,
+    })
+    .catch(() => null);
+
+  if (!role) {
+    console.info("Creating role", roleName);
+    await iam.createRole({
+      RoleName: roleName,
+      Path: "/",
+      Description: "aws sdk js v3 lambda test role",
+      AssumeRolePolicyDocument: JSON.stringify({
+        Version: "2012-10-17",
+        Statement: [
+          {
+            Effect: "Allow",
+            Action: ["sts:AssumeRole"],
+            Principal: {
+              Service: ["lambda.amazonaws.com"],
+            },
+          },
+        ],
+      }),
+    });
+  } else {
+    console.info("Role exists", roleName);
+  }
+
+  const listAttachedRolePolicies = await iam.listAttachedRolePolicies({
+    RoleName: roleName,
+  });
+  const policies = listAttachedRolePolicies.AttachedPolicies || [];
+
+  const existingPolicies = policies.reduce((acc: Record<string, boolean>, cur: AttachedPolicy) => {
+    if (cur.PolicyName) {
+      acc[cur.PolicyName] = true;
+    }
+    return acc;
+  }, {} as Record<string, boolean>);
+
+  const required = ["AWSLambda_FullAccess"];
+
+  for (const requiredPolicy of required) {
+    if (!existingPolicies[requiredPolicy]) {
+      console.info("Attaching policy to role", requiredPolicy, roleName);
+      await iam.attachRolePolicy({
+        RoleName: roleName,
+        PolicyArn: `arn:aws:iam::aws:policy/${requiredPolicy}`,
+      });
+    } else {
+      console.info("Policy exists on role", requiredPolicy, roleName);
+    }
+  }
+
+  const getRole: null | GetRoleResponse = await iam
+    .getRole({
+      RoleName: roleName,
+    })
+    .catch(() => null);
+  if (!getRole) {
+    throw new Error("Role not found.");
+  } else {
+    console.info("Role found", roleName);
+  }
+
+  const roleArn = getRole.Role!.Arn!;
+
+  if (getFn) {
+    console.info("Function exists:", FunctionName);
+
+    if ((getFn.Timeout ?? 0) < 5 * 60 || getFn?.Handler !== Handler) {
+      await lambda.updateFunctionConfiguration({
+        FunctionName,
+        Handler,
+        Timeout: 5 * 60,
+      });
+      await waitUntilFunctionUpdated(
+        {
+          client: lambda,
+          maxWaitTime: 40,
+        },
+        {
+          FunctionName,
+        }
+      );
+    }
+    // await lambda.updateFunctionCode({
+    //   FunctionName,
+    //   ZipFile: fs.readFileSync(require.resolve("./function.zip")),
+    // });
+    // console.info("Function code/configuration updated:", FunctionName);
+  } else {
+    await lambda.createFunction({
+      FunctionName,
+      Role: roleArn,
+      Code: {
+        ZipFile: fs.readFileSync(require.resolve("./function.zip")),
+      },
+      Runtime: Runtime.nodejs16x,
+      Description: `aws sdk js v3 e2e test echo`,
+      Timeout: 300,
+      Handler,
+    });
+    console.info("Function created:", FunctionName);
+  }
+
+  await waitUntilFunctionActiveV2(
+    {
+      maxWaitTime: 40,
+      client: lambda,
+    },
+    {
+      FunctionName,
+    }
+  );
+}

packages/util-stream/test/util-stream-blob.e2e.spec.ts

@@ -0,0 +1,56 @@
+import { Lambda } from "@aws-sdk/client-lambda";
+import { Uint8ArrayBlobAdapter } from "@aws-sdk/util-stream";
+import { Readable } from "stream";
+
+import { FunctionName, setup } from "./setup";
+
+describe("blob e2e", () => {
+  jest.setTimeout(100000);
+
+  const lambda = new Lambda({
+    region: "us-west-2",
+  });
+
+  beforeAll(async () => {
+    await setup();
+  });
+
+  it("should allow string as payload blob and allow conversion of output payload blob to string", async () => {
+    const payload = JSON.stringify({ hello: "world" });
+    const invoke = await lambda.invoke({ FunctionName, Payload: payload });
+    expect(JSON.parse(invoke?.Payload?.transformToString() ?? "{}")).toEqual({ hello: "world" });
+  });
+
+  it("should allow Uint8Array as payload blob", async () => {
+    const payload = Uint8ArrayBlobAdapter.fromString(JSON.stringify({ hello: "world" }));
+    const invoke = await lambda.invoke({ FunctionName, Payload: payload });
+    expect(JSON.parse(invoke?.Payload?.transformToString() ?? "{}")).toEqual({ hello: "world" });
+  });
+
+  it("should allow buffer as payload blob", async () => {
+    // note: Buffer extends Uint8Array
+    const payload = Buffer.from(Uint8ArrayBlobAdapter.fromString(JSON.stringify({ hello: "world" })));
+    const invoke = await lambda.invoke({ FunctionName, Payload: payload });
+    expect(JSON.parse(invoke?.Payload?.transformToString() ?? "{}")).toEqual({ hello: "world" });
+  });
+
+  it("should allow stream as payload blob but not be able to sign it", async () => {
+    const payload = Readable.from(Buffer.from(Uint8ArrayBlobAdapter.fromString(JSON.stringify({ hello: "world" }))), {
+      encoding: "utf-8",
+    });
+    expect(JSON.parse(await streamToString(payload))).toEqual({ hello: "world" });
+    await lambda.invoke({ FunctionName, Payload: payload }).catch((e) => {
+      expect(e.toString()).toContain("InvalidSignatureException");
+    });
+    expect.hasAssertions();
+  });
+});
+
+function streamToString(stream: Readable): Promise<string> {
+  const chunks: any[] = [];
+  return new Promise((resolve, reject) => {
+    stream.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
+    stream.on("error", (err) => reject(err));
+    stream.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
+  });
+}