chore(layers): bundle assets from source when testing (#1710)

Author: dreamorosi

.github/scripts/setup_tmp_layer_files.sh

@@ -42,10 +42,6 @@ jobs:
           node-version: "18"
       - name: Setup dependencies
         uses: ./.github/actions/cached-node-modules
-      - name: Create layer files
-        run: |
-          export VERSION=${{ inputs.latest_published_version }}
-          bash .github/scripts/setup_tmp_layer_files.sh
       - name: CDK build
         run: npm run cdk -w layers -- synth --context PowertoolsPackageVersion=${{ inputs.latest_published_version }} -o cdk.out
       - name: Zip output

.github/workflows/publish_layer.yml

@@ -69,10 +69,6 @@ jobs:
         uses: ./.github/actions/cached-node-modules
       - name: Run linting
         run: npm run lint -w layers
-      - name: Create layer files
-        run: |
-          export VERSION=latest
-          bash .github/scripts/setup_tmp_layer_files.sh
       - name: Run tests
         run: npm run test:unit -w layers
   check-docs-snippets:

.github/workflows/reusable-run-linting-check-and-unit-tests.yml

@@ -55,11 +55,6 @@ jobs:
           role-to-assume: ${{ secrets.AWS_ROLE_ARN_TO_ASSUME }}
           aws-region: eu-west-1
           mask-aws-account-id: true
-      - name: Create layer files
-        if: ${{ matrix.package == 'layers' }}
-        run: |
-          export VERSION=latest
-          bash .github/scripts/setup_tmp_layer_files.sh
       - name: Run integration tests on utils
         env:
           RUNTIME: nodejs${{ matrix.version }}x

.github/workflows/run-e2e-tests.yml

@@ -44,10 +44,6 @@ RUNTIME=node12.x VERSION=0.9.0 npm run test:e2e
 ```shell
  cdk bootstrap aws://AWS_ACCOUNT/NEW_REGION   
 ```
-* Build the layer folder from the project root directory
-```shell
-bash ./.github/scripts/setup_tmp_layer_files.sh 
-```
 * Deploy the first layer version to the new region, make sure to set the NEW_REGION in your AWS CLI configuration correctly, otherwise you will deploy to the wrong region
 ```shell
 npm run cdk -w layers -- deploy --app cdk.out --context region=NEW_REGION 'LayerPublisherStack' --require-approval never --verbose 

layers/README.md

@@ -10,6 +10,7 @@ const app = new App();
 
 new LayerPublisherStack(app, 'LayerPublisherStack', {
   powertoolsPackageVersion: app.node.tryGetContext('PowertoolsPackageVersion'),
+  buildFromLocal: app.node.tryGetContext('BuildFromLocal') || false,
   layerName: 'AWSLambdaPowertoolsTypeScript',
   ssmParameterLayerArn: SSM_PARAM_LAYER_ARN,
 });

layers/bin/layers.ts

@@ -14,7 +14,8 @@
     "lint": "eslint --ext .ts,.js --no-error-on-unmatched-pattern .",
     "lint-fix": "eslint --fix --ext .ts,.js --fix --no-error-on-unmatched-pattern .",
     "test:unit": "jest --group=unit",
-    "test:e2e": "jest --group=e2e"
+    "test:e2e": "jest --group=e2e",
+    "createLayerFolder": "cdk synth --context BuildFromLocal=true"
   },
   "lint-staged": {
     "*.{js,ts}": "npm run lint-fix"

layers/package.json

@@ -1,12 +1,12 @@
 import { CustomResource, Duration, Stack, StackProps } from 'aws-cdk-lib';
 import { Construct } from 'constructs';
-import { LayerVersion, Runtime } from 'aws-cdk-lib/aws-lambda';
+import { LayerVersion, Runtime, Tracing } from 'aws-cdk-lib/aws-lambda';
 import { RetentionDays } from 'aws-cdk-lib/aws-logs';
 import { randomUUID } from 'node:crypto';
 import { Effect, PolicyStatement } from 'aws-cdk-lib/aws-iam';
 import { Provider } from 'aws-cdk-lib/custom-resources';
 import { StringParameter } from 'aws-cdk-lib/aws-ssm';
-import path from 'path';
+import path from 'node:path';
 import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs';
 
 export interface CanaryStackProps extends StackProps {
@@ -54,13 +54,15 @@ export class CanaryStack extends Stack {
         ],
       },
       environment: {
+        LAYERS_PATH: '/opt/nodejs/node_modules',
         POWERTOOLS_SERVICE_NAME: 'canary',
         POWERTOOLS_PACKAGE_VERSION: powertoolsPackageVersion,
         POWERTOOLS_LAYER_NAME: layerName,
         SSM_PARAMETER_LAYER_ARN: props.ssmParameterLayerArn,
       },
       layers: layer,
       logRetention: RetentionDays.ONE_DAY,
+      tracing: Tracing.ACTIVE,
     });
 
     canaryFunction.addToRolePolicy(

layers/src/canary-stack.ts

@@ -1,18 +1,21 @@
 import { CfnOutput, RemovalPolicy, Stack, StackProps } from 'aws-cdk-lib';
-import { Construct } from 'constructs';
 import {
-  LayerVersion,
+  CfnLayerVersionPermission,
   Code,
+  LayerVersion,
   Runtime,
-  CfnLayerVersionPermission,
 } from 'aws-cdk-lib/aws-lambda';
 import { StringParameter } from 'aws-cdk-lib/aws-ssm';
-import { resolve } from 'node:path';
+import { Construct } from 'constructs';
+import { execSync } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { join, resolve, sep } from 'node:path';
 
 export interface LayerPublisherStackProps extends StackProps {
   readonly layerName?: string;
   readonly powertoolsPackageVersion?: string;
   readonly ssmParameterLayerArn: string;
+  readonly buildFromLocal?: boolean;
 }
 
 export class LayerPublisherStack extends Stack {
@@ -24,7 +27,7 @@ export class LayerPublisherStack extends Stack {
   ) {
     super(scope, id, props);
 
-    const { layerName, powertoolsPackageVersion } = props;
+    const { layerName, powertoolsPackageVersion, buildFromLocal } = props;
 
     console.log(
       `publishing layer ${layerName} version : ${powertoolsPackageVersion}`
@@ -41,7 +44,112 @@ export class LayerPublisherStack extends Stack {
       license: 'MIT-0',
       // This is needed because the following regions do not support the compatibleArchitectures property #1400
       // ...(![ 'eu-south-2', 'eu-central-2', 'ap-southeast-4' ].includes(Stack.of(this).region) ? { compatibleArchitectures: [Architecture.X86_64] } : {}),
-      code: Code.fromAsset(resolve(__dirname, '..', '..', 'tmp')),
+      code: Code.fromAsset(resolve(__dirname), {
+        bundling: {
+          // This is here only because is required by CDK, however it is not used since the bundling is done locally
+          image: Runtime.NODEJS_18_X.bundlingImage,
+          // We need to run a command to generate a random UUID to force the bundling to run every time
+          command: [`echo "${randomUUID()}"`],
+          local: {
+            tryBundle(outputDir: string) {
+              // This folder are relative to the layers folder
+              const tmpBuildPath = resolve(__dirname, '..', 'tmp');
+              const tmpBuildDir = join(tmpBuildPath, 'nodejs');
+              // This folder is the project root, relative to the current file
+              const projectRoot = resolve(__dirname, '..', '..');
+
+              // This is the list of packages that we need include in the Lambda Layer
+              // the name is the same as the npm workspace name
+              const utilities = ['commons', 'logger', 'metrics', 'tracer'];
+
+              // These files are relative to the tmp folder
+              const filesToRemove = [
+                'node_modules/@types',
+                'package.json',
+                'package-lock.json',
+                'node_modules/**/README.md',
+                'node_modules/.bin/semver',
+                'node_modules/async-hook-jl/test',
+                'node_modules/shimmer/test',
+                'node_modules/jmespath/artifacts',
+                // We remove the type definitions since they can't be used in the Lambda Layer
+                'node_modules/@aws-lambda-powertools/*/lib/*.d.ts',
+                'node_modules/@aws-lambda-powertools/*/lib/*.d.ts.map',
+              ];
+              const buildCommands: string[] = [];
+              const modulesToInstall: string[] = [];
+
+              if (buildFromLocal) {
+                for (const util of utilities) {
+                  // Build latest version of the package
+                  buildCommands.push(`npm run build -w packages/${util}`);
+                  // Pack the package to a .tgz file
+                  buildCommands.push(`npm pack -w packages/${util}`);
+                  // Move the .tgz file to the tmp folder
+                  buildCommands.push(
+                    `mv aws-lambda-powertools-${util}-*.tgz ${tmpBuildDir}`
+                  );
+                }
+                modulesToInstall.push(
+                  ...utilities.map((util) =>
+                    join(tmpBuildDir, `aws-lambda-powertools-${util}-*.tgz`)
+                  )
+                );
+                filesToRemove.push(
+                  ...utilities.map((util) =>
+                    join(`aws-lambda-powertools-${util}-*.tgz`)
+                  )
+                );
+              } else {
+                // Dependencies to install in the Lambda Layer
+                modulesToInstall.push(
+                  ...utilities.map(
+                    (util) =>
+                      `@aws-lambda-powertools/${util}@${powertoolsPackageVersion}`
+                  )
+                );
+              }
+
+              // Phase 1: Cleanup & create tmp folder
+              execSync(
+                [
+                  // Clean up existing tmp folder from previous builds
+                  `rm -rf ${tmpBuildDir}`,
+                  // Create tmp folder again
+                  `mkdir -p ${tmpBuildDir}`,
+                ].join(' && ')
+              );
+
+              // Phase 2: (Optional) Build packages & pack them
+              buildFromLocal &&
+                execSync(buildCommands.join(' && '), { cwd: projectRoot });
+
+              // Phase 3: Install dependencies to tmp folder
+              execSync(
+                `npm i --prefix ${tmpBuildDir} ${modulesToInstall.join(' ')}`
+              );
+
+              // Phase 4: Remove unnecessary files
+              execSync(
+                `rm -rf ${filesToRemove
+                  .map((filePath) => `${tmpBuildDir}/${filePath}`)
+                  .join(' ')}`
+              );
+
+              // Phase 5: Copy files from tmp folder to cdk.out asset folder (the folder is created by CDK)
+              execSync(`cp -R ${tmpBuildPath}${sep}* ${outputDir}`);
+
+              // Phase 6: (Optional) Restore changes to the project root made by the build
+              buildFromLocal &&
+                execSync('git restore packages/*/package.json', {
+                  cwd: projectRoot,
+                });
+
+              return true;
+            },
+          },
+        },
+      }),
     });
 
     const layerPermission = new CfnLayerVersionPermission(

layers/src/layer-publisher-stack.ts

@@ -1,4 +1,5 @@
-import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { readFile } from 'node:fs/promises';
 import { Logger } from '@aws-lambda-powertools/logger';
 import { Metrics } from '@aws-lambda-powertools/metrics';
 import { Tracer } from '@aws-lambda-powertools/tracer';
@@ -9,36 +10,63 @@ const logger = new Logger({
 const metrics = new Metrics();
 const tracer = new Tracer();
 
-export const handler = (): void => {
-  // Check that the packages version matches the expected one
-  const packageJSON = JSON.parse(
-    readFileSync(
-      '/opt/nodejs/node_modules/@aws-lambda-powertools/logger/package.json',
-      {
-        encoding: 'utf8',
-        flag: 'r',
-      }
-    )
+const layerPath = process.env.LAYERS_PATH || '/opt/nodejs/node_modules';
+const expectedVersion = process.env.POWERTOOLS_PACKAGE_VERSION || '0.0.0';
+
+const getVersionFromModule = async (moduleName: string): Promise<string> => {
+  const manifestPath = join(
+    layerPath,
+    '@aws-lambda-powertools',
+    moduleName,
+    'package.json'
   );
 
-  if (packageJSON.version != process.env.POWERTOOLS_PACKAGE_VERSION) {
-    throw new Error(
-      `Package version mismatch: ${packageJSON.version} != ${process.env.POWERTOOLS_PACKAGE_VERSION}`
-    );
+  let manifest: string;
+  try {
+    manifest = await readFile(manifestPath, { encoding: 'utf8' });
+  } catch (error) {
+    console.log(error);
+    throw new Error(`Unable to read/find package.json file at ${manifestPath}`);
   }
 
-  // Check that the logger is working
-  logger.debug('Hello World!');
+  let moduleVersion: string;
+  try {
+    const { version } = JSON.parse(manifest);
+    moduleVersion = version;
+  } catch (error) {
+    console.log(error);
+    throw new Error(`Unable to parse package.json file at ${manifestPath}`);
+  }
+
+  return moduleVersion;
+};
+
+export const handler = async (): Promise<void> => {
+  // Check that the packages version matches the expected one
+  for (const moduleName of ['commons', 'logger', 'metrics', 'tracer']) {
+    const moduleVersion = await getVersionFromModule(moduleName);
+    if (moduleVersion != expectedVersion) {
+      throw new Error(
+        `Package version mismatch (${moduleName}): ${moduleVersion} != ${expectedVersion}`
+      );
+    }
+  }
 
   // Check that the metrics is working
   metrics.captureColdStartMetric();
 
   // Check that the tracer is working
-  const segment = tracer.getSegment();
-  if (!segment) throw new Error('Segment not found');
-  const handlerSegment = segment.addNewSubsegment('### index.handler');
-  tracer.setSegment(handlerSegment);
+  const subsegment = tracer.getSegment()?.addNewSubsegment('### index.handler');
+  if (!subsegment) {
+    throw new Error('Unable to create subsegment, check the Tracer');
+  }
+  tracer.setSegment(subsegment);
   tracer.annotateColdStart();
-  handlerSegment.close();
-  tracer.setSegment(segment);
+  subsegment.close();
+  tracer.setSegment(subsegment.parent);
+
+  // Check that logger & tracer are both working
+  // the presence of a log will indicate that the logger is working
+  // while the content of the log will indicate that the tracer is working
+  logger.debug('subsegment', { subsegment: subsegment.format() });
 };