From 0399c6c0bf70dc36eeee05e327162a90555f6393 Mon Sep 17 00:00:00 2001 From: mouhib Date: Sun, 16 Apr 2023 18:31:58 +0100 Subject: [PATCH 1/6] add e2e action --- .github/workflows/e2e.yml | 47 +++++++++++++++++++ .../test/e2e/synchronous-athena-query.test.ts | 2 +- core/test/e2e/synchronous-crawler.test.ts | 2 +- core/test/e2e/synchronous-glue-job.test.ts | 2 +- 4 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..a6c1cfe8 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,47 @@ +name: E2E +on: + pull_request: {} +jobs: + build: + runs-on: ubuntu-latest + permissions: + id-token: write # needed to interact with GitHub's OIDC Token endpoint. + contents: read + strategy: + matrix: + suite: [data-lake, synchronous, emr-eks-cluster] + fail-fast: false + env: + CI: "true" + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + - name: Setup Node 14 + uses: actions/setup-node@v2 + with: + node-version: '14' + - name: Setup Java 11 + uses: actions/setup-java@v2 + with: + distribution: 'zulu' # OpenJDK + java-version: '11' + - name: Install dependencies + run: cd core && yarn install --check-files --frozen-lockfile + - name: Set git identity + run: |- + git config user.name "Automation" + git config user.email "github-actions@github.com" + - name: Build + run: cd core && npx projen build + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1.6.1 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN_TO_ASSUME }} + aws-region: us-east-1 + - name: E2E + run: cd core && npx jest --group=integ/${{ matrix.suite }}/ + container: + image: jsii/superchain:1-buster-slim-node14 diff --git a/core/test/e2e/synchronous-athena-query.test.ts b/core/test/e2e/synchronous-athena-query.test.ts index 8ed67983..e8539e16 100644 --- a/core/test/e2e/synchronous-athena-query.test.ts +++ b/core/test/e2e/synchronous-athena-query.test.ts @@ -4,7 +4,7 @@ /** * Tests SynchronousAthenaQuery * - * @group integ/synchronous-athena-query + * @group integ/synchronous/synchronous-athena-query */ import { PolicyStatement } from 'aws-cdk-lib/aws-iam'; diff --git a/core/test/e2e/synchronous-crawler.test.ts b/core/test/e2e/synchronous-crawler.test.ts index 5b0844df..0d04f63a 100644 --- a/core/test/e2e/synchronous-crawler.test.ts +++ b/core/test/e2e/synchronous-crawler.test.ts @@ -4,7 +4,7 @@ /** * Tests SyncrhonousCrawler * - * @group integ/synchronous-crawler + * @group integ/synchronous/synchronous-crawler */ import * as cdk from 'aws-cdk-lib'; diff --git a/core/test/e2e/synchronous-glue-job.test.ts b/core/test/e2e/synchronous-glue-job.test.ts index 92aa681f..0ec8dd6f 100644 --- a/core/test/e2e/synchronous-glue-job.test.ts +++ b/core/test/e2e/synchronous-glue-job.test.ts @@ -4,7 +4,7 @@ /** * Tests SynchronousGlueJob * - * @group integ/synchronous-glue-job + * @group integ/synchronous/synchronous-glue-job */ import * as cdk from 'aws-cdk-lib'; From d262b2fee2ca157137edbec084124e49339240ce Mon Sep 17 00:00:00 2001 From: mouhib Date: Sun, 16 Apr 2023 18:38:28 +0100 Subject: [PATCH 2/6] add trigger --- .github/workflows/e2e.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a6c1cfe8..fe6bfeb4 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,6 +1,8 @@ name: E2E on: - pull_request: {} + push: + branches: + - "test/e2e" jobs: build: runs-on: ubuntu-latest From d831bc74fde864b7bdb2660ee4200aa391326c6a Mon Sep 17 00:00:00 2001 From: mouhib Date: Sun, 16 Apr 2023 19:11:41 +0100 Subject: [PATCH 3/6] update e2e test --- .github/workflows/e2e.yml | 4 +- .../notebook-platform/notebook-platform.ts | 1 + .../e2e/emr-eks-notebook-platform.test.ts | 53 +++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 core/test/e2e/emr-eks-notebook-platform.test.ts diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index fe6bfeb4..d06a8035 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -44,6 +44,8 @@ jobs: role-to-assume: ${{ secrets.AWS_ROLE_ARN_TO_ASSUME }} aws-region: us-east-1 - name: E2E - run: cd core && npx jest --group=integ/${{ matrix.suite }}/ + run: cd core && npx jest --group=integ/${{ matrix.suite }} + - name: E2E-NOTEBOOK + run: cd core && npx jest --group=integ/notebook-platform container: image: jsii/superchain:1-buster-slim-node14 diff --git a/core/src/notebook-platform/notebook-platform.ts b/core/src/notebook-platform/notebook-platform.ts index b3bf418e..419d87ac 100644 --- a/core/src/notebook-platform/notebook-platform.ts +++ b/core/src/notebook-platform/notebook-platform.ts @@ -333,6 +333,7 @@ export class NotebookPlatform extends TrackedConstruct { new CfnOutput(this, `URL for EMR Studio: ${this.studioName}`, { value: this.studioInstance.attrUrl, + exportName: `{this.studioName}` }); /*//Return EMR Studio URL as CfnOutput diff --git a/core/test/e2e/emr-eks-notebook-platform.test.ts b/core/test/e2e/emr-eks-notebook-platform.test.ts new file mode 100644 index 00000000..e008d07b --- /dev/null +++ b/core/test/e2e/emr-eks-notebook-platform.test.ts @@ -0,0 +1,53 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT-0 + +/** + * Tests EmrEksCluster + * + * @group integ/notebook-platform + */ + +import * as cdk from 'aws-cdk-lib'; +import { deployStack, destroyStack } from './utils'; + +import { Autoscaler, EmrEksCluster } from '../../src/emr-eks-platform'; +import { NotebookPlatform, StudioAuthMode } from '../../src/notebook-platform'; + +jest.setTimeout(2000000); +// GIVEN +const integTestApp = new cdk.App(); +const stack = new cdk.Stack(integTestApp, 'EmrEksClustereE2eTest'); + +const emrEksCluster = EmrEksCluster.getOrCreate(stack, { + eksAdminRoleArn: 'arn:aws:iam::123445678912:role/my-role', + autoscaling: Autoscaler.CLUSTER_AUTOSCALER, +}); + +let iamAuthNotebook = new NotebookPlatform(stack, 'dataplatformIAMAuth', { + emrEks: emrEksCluster, + studioName: 'e2estudio', + studioAuthMode: StudioAuthMode.IAM, + eksNamespace: 'dataplatformiamauth', + }); + +new cdk.CfnOutput(stack, 'EmrEksAdminRoleOutput', { + value: emrEksCluster.eksCluster.adminRole.roleArn, + exportName: 'emrEksAdminRole', +}); + +describe('deploy succeed', () => { + it('can be deploy succcessfully', async () => { + // GIVEN + const deployResult = await deployStack(integTestApp, stack); + + // THEN + expect(deployResult.outputs.emrEksAdminRole).toEqual('arn:aws:iam::123445678912:role/my-role'); + + expect(deployResult.outputs.e2estudio).toMatch(/^https:\/\/.*/); + + }, 9000000); +}); + +afterAll(async () => { + await destroyStack(integTestApp, stack); +}, 9000000); From 0ca4c1bfb7a26f2d8f8ac217c228aca867687c70 Mon Sep 17 00:00:00 2001 From: mouhib Date: Sun, 16 Apr 2023 19:14:42 +0100 Subject: [PATCH 4/6] fix e2e --- .github/workflows/e2e.yml | 2 -- core/test/e2e/emr-eks-notebook-platform.test.ts | 2 +- core/test/e2e/emr-eks.test.ts | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d06a8035..bd9b3fc2 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -45,7 +45,5 @@ jobs: aws-region: us-east-1 - name: E2E run: cd core && npx jest --group=integ/${{ matrix.suite }} - - name: E2E-NOTEBOOK - run: cd core && npx jest --group=integ/notebook-platform container: image: jsii/superchain:1-buster-slim-node14 diff --git a/core/test/e2e/emr-eks-notebook-platform.test.ts b/core/test/e2e/emr-eks-notebook-platform.test.ts index e008d07b..a182e35c 100644 --- a/core/test/e2e/emr-eks-notebook-platform.test.ts +++ b/core/test/e2e/emr-eks-notebook-platform.test.ts @@ -4,7 +4,7 @@ /** * Tests EmrEksCluster * - * @group integ/notebook-platform + * @group integ/emr-eks/notebook-platform */ import * as cdk from 'aws-cdk-lib'; diff --git a/core/test/e2e/emr-eks.test.ts b/core/test/e2e/emr-eks.test.ts index af3ecd73..3d2e926f 100644 --- a/core/test/e2e/emr-eks.test.ts +++ b/core/test/e2e/emr-eks.test.ts @@ -4,7 +4,7 @@ /** * Tests EmrEksCluster * - * @group integ/emr-eks-cluster + * @group integ/emr-eks/cluster */ import * as cdk from 'aws-cdk-lib'; From 5b7d7c4002f0c224121a1728adfa4e945e3270c2 Mon Sep 17 00:00:00 2001 From: mouhib Date: Sun, 16 Apr 2023 19:40:33 +0100 Subject: [PATCH 5/6] fix e2e test --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index bd9b3fc2..ec4f3f39 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -11,7 +11,7 @@ jobs: contents: read strategy: matrix: - suite: [data-lake, synchronous, emr-eks-cluster] + suite: [data-lake, synchronous, emr-eks] fail-fast: false env: CI: "true" From b45eccbaa5e2c167809d0a8c178aa48fb960f57d Mon Sep 17 00:00:00 2001 From: mouhib Date: Sun, 16 Apr 2023 19:51:48 +0100 Subject: [PATCH 6/6] fix lint error for notebook e2e --- core/test/e2e/emr-eks-notebook-platform.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/test/e2e/emr-eks-notebook-platform.test.ts b/core/test/e2e/emr-eks-notebook-platform.test.ts index a182e35c..5f6eb3f9 100644 --- a/core/test/e2e/emr-eks-notebook-platform.test.ts +++ b/core/test/e2e/emr-eks-notebook-platform.test.ts @@ -23,7 +23,7 @@ const emrEksCluster = EmrEksCluster.getOrCreate(stack, { autoscaling: Autoscaler.CLUSTER_AUTOSCALER, }); -let iamAuthNotebook = new NotebookPlatform(stack, 'dataplatformIAMAuth', { +new NotebookPlatform(stack, 'dataplatformIAMAuth', { emrEks: emrEksCluster, studioName: 'e2estudio', studioAuthMode: StudioAuthMode.IAM,