diff --git a/.github/workflows/integ.yml b/.github/workflows/integ.yml index 164d1cb..8047a24 100644 --- a/.github/workflows/integ.yml +++ b/.github/workflows/integ.yml @@ -76,8 +76,8 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 - role-duration-seconds: 14400 - role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME_FOR_TESTING }} + role-duration-seconds: 3600 + role-to-assume: ${{ vars.CDK_ATMOSPHERE_PROD_OIDC_ROLE }} role-session-name: run-tests@aws-cdk-cli-integ output-credentials: true - name: Set git identity @@ -118,12 +118,13 @@ jobs: echo "lib_version=${LIB_VERSION}" >> $GITHUB_OUTPUT - name: "Run the test suite: ${{ matrix.suite }}" env: - JEST_TEST_CONCURRENT: ${{ matrix.suite == 'cli-integ-tests' && 'true' || 'false' }} JSII_SILENCE_WARNING_DEPRECATED_NODE_VERSION: "true" JSII_SILENCE_WARNING_UNTESTED_NODE_VERSION: "true" JSII_SILENCE_WARNING_KNOWN_BROKEN_NODE_VERSION: "true" DOCKERHUB_DISABLED: "true" - AWS_REGIONS: us-east-2,eu-west-1,eu-north-1,ap-northeast-1,ap-south-1 + CDK_INTEG_ATMOSPHERE_ENABLED: "true" + CDK_INTEG_ATMOSPHERE_ENDPOINT: ${{ vars.CDK_ATMOSPHERE_PROD_ENDPOINT }} + CDK_INTEG_ATMOSPHERE_POOL: ${{ vars.CDK_INTEG_ATMOSPHERE_POOL }} CDK_MAJOR_VERSION: "2" RELEASE_TAG: latest GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.projenrc.ts b/.projenrc.ts index b2a6a88..6f66953 100644 --- a/.projenrc.ts +++ b/.projenrc.ts @@ -240,6 +240,11 @@ new CdkCliIntegTestsWorkflow(repo, { testEnvironment: TEST_ENVIRONMENT, testRunsOn: TEST_RUNNER, localPackages: [cliInteg.name], + enableAtmosphere: { + oidcRoleArn: '${{ vars.CDK_ATMOSPHERE_PROD_OIDC_ROLE }}', + endpoint: '${{ vars.CDK_ATMOSPHERE_PROD_ENDPOINT }}', + pool: '${{ vars.CDK_INTEG_ATMOSPHERE_POOL }}' + } }); repo.synth(); diff --git a/package.json b/package.json index 54dbca2..d2da5f7 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "@types/node": "^22.13.10", "@typescript-eslint/eslint-plugin": "^8", "@typescript-eslint/parser": "^8", - "cdklabs-projen-project-types": "^0.2.4", + "cdklabs-projen-project-types": "^0.2.13", "constructs": "^10.0.0", "eslint-plugin-import": "^2.31.0", "eslint-plugin-jest": "^28.11.0", diff --git a/packages/@aws-cdk-testing/cli-integ/.projen/tasks.json b/packages/@aws-cdk-testing/cli-integ/.projen/tasks.json index cfc5ecd..778e9e2 100644 --- a/packages/@aws-cdk-testing/cli-integ/.projen/tasks.json +++ b/packages/@aws-cdk-testing/cli-integ/.projen/tasks.json @@ -92,7 +92,7 @@ "name": "gather-versions", "steps": [ { - "exec": "node -e \"require(path.join(path.dirname(require.resolve('cdklabs-projen-project-types')), 'yarn', 'gather-versions.exec.js'))\" @aws-cdk-testing/cli-integ MAJOR --deps ", + "exec": "node -e \"require(require.resolve('cdklabs-projen-project-types/lib/yarn/gather-versions.exec.js')).cliMain()\" ", "receiveArgs": true } ] @@ -175,7 +175,10 @@ "builtin": "release/reset-version" }, { - "spawn": "gather-versions" + "spawn": "gather-versions", + "env": { + "RESET_VERSIONS": "true" + } } ] }, diff --git a/packages/@aws-cdk-testing/cli-integ/lib/aws.ts b/packages/@aws-cdk-testing/cli-integ/lib/aws.ts index f23f492..79b23e6 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/aws.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/aws.ts @@ -80,6 +80,20 @@ export class AwsClients { return (await stsClient.send(new GetCallerIdentityCommand({}))).Account!; } + /** + * If the clients already has an established identity (via atmosphere for example), + * return an environment variable map activating it. + * + * Otherwise, returns undefined. + */ + public identityEnv(): Record | undefined { + return this.identity ? { + AWS_ACCESS_KEY_ID: this.identity.accessKeyId, + AWS_SECRET_ACCESS_KEY: this.identity.secretAccessKey, + AWS_SESSION_TOKEN: this.identity.sessionToken!, + } : undefined; + } + /** * Resolve the current identity or identity provider to credentials */ diff --git a/packages/@aws-cdk-testing/cli-integ/lib/shell.ts b/packages/@aws-cdk-testing/cli-integ/lib/shell.ts index f63fec9..5897fe5 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/shell.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/shell.ts @@ -60,9 +60,13 @@ export async function shell(command: string[], options: ShellOptions = {}): Prom if (interaction) { if (interaction.prompt.test(lastLine.get())) { - // subprocess expects a user input now - child.writeStdin(interaction.input + (interaction.end ?? os.EOL)); - remainingInteractions.shift(); + // subprocess expects a user input now. + // we have to write the input AFTER the child has started + // reading, so we do this with a small delay. + setTimeout(() => { + child.writeStdin(interaction.input + (interaction.end ?? os.EOL)); + remainingInteractions.shift(); + }, 500); } } @@ -218,6 +222,12 @@ export class ShellHelper { outputs: [this._output], cwd: this._cwd, ...options, + modEnv: { + // give every shell its own docker config directory + // so that parallel runs don't interfere with each other. + DOCKER_CONFIG: path.join(this._cwd, '.docker'), + ...options.modEnv, + }, }); } } diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts index d658fcb..39a9027 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts @@ -4,7 +4,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { DescribeStacksCommand, Stack } from '@aws-sdk/client-cloudformation'; -import { outputFromStack, AwsClients } from './aws'; +import { outputFromStack, AwsClients, sleep } from './aws'; import { TestContext } from './integ-test'; import { findYarnPackages } from './package-sources/repo-source'; import { IPackageSource } from './package-sources/source'; @@ -525,11 +525,7 @@ export class TestFixture extends ShellHelper { public cdkShellEnv() { // if tests are using an explicit aws identity already (i.e creds) // force every cdk command to use the same identity. - const awsCreds: Record = this.aws.identity ? { - AWS_ACCESS_KEY_ID: this.aws.identity.accessKeyId, - AWS_SECRET_ACCESS_KEY: this.aws.identity.secretAccessKey, - AWS_SESSION_TOKEN: this.aws.identity.sessionToken!, - } : {}; + const awsCreds = this.aws.identityEnv() ?? {}; return { AWS_REGION: this.aws.region, @@ -693,17 +689,61 @@ export async function ensureBootstrapped(fixture: TestFixture) { const envSpecifier = `aws://${await fixture.aws.account()}/${fixture.aws.region}`; if (ALREADY_BOOTSTRAPPED_IN_THIS_RUN.has(envSpecifier)) { return; } - await fixture.cdk(['bootstrap', envSpecifier], { + if (atmosphereEnabled()) { + // when atmosphere is enabled, each test starts with an empty environment + // and needs to deploy the bootstrap stack. in case environments are recylced too quickly, + // cloudformation may think the bootstrap bucket still exists even though it doesnt (because of s3 eventual consistency). + // so we retry on the specific error for a while. + await bootstrapWithRetryOnBucketExists(envSpecifier, fixture); + } else { + await doBootstrap(envSpecifier, fixture, false); + } + + // when using the atmosphere service, every test needs to bootstrap + // its own environment. + if (!atmosphereEnabled()) { + ALREADY_BOOTSTRAPPED_IN_THIS_RUN.add(envSpecifier); + } +} + +async function doBootstrap(envSpecifier: string, fixture: TestFixture, allowErrExit: boolean) { + return fixture.cdk(['bootstrap', envSpecifier], { modEnv: { // Even for v1, use new bootstrap CDK_NEW_BOOTSTRAP: '1', + // when allowing error exit, we probably want to inspect + // and compare output, which is better done without color characters. + ...(allowErrExit ? { FORCE_COLOR: '0' } : {}), }, + allowErrExit, }); +} - // when using the atmosphere service, every test needs to bootstrap - // its own environment. - if (!atmosphereEnabled()) { - ALREADY_BOOTSTRAPPED_IN_THIS_RUN.add(envSpecifier); +async function bootstrapWithRetryOnBucketExists(envSpecifier: string, fixture: TestFixture) { + + const account = await fixture.aws.account(); + const retryAfterSeconds = 30; + const bootstrapBucket = `cdk-hnb659fds-assets-${account}-${fixture.aws.region}`; + + // s3 says that a bucket deletion can take up to an hour to be fully visible. + // empirically we see that a few minutes is enough though. lets give 10 to be on the safe(r) side. + const timeoutMinutes = 10; + + const timeoutDate = new Date(Date.now() + timeoutMinutes * 60 * 1000) + while (true) { + const out = await doBootstrap(envSpecifier, fixture, true); + if (out.includes(`Environment ${envSpecifier} bootstrapped`)) { + break; + } + if (out.includes(`${bootstrapBucket} already exists`)) { + // might be an s3 eventualy consistency issue due to recycled environments. + if (Date.now() < timeoutDate.getTime()) { + fixture.log(`Bootstrap of ${envSpecifier} failed due to bucket existence check. Retrying in ${retryAfterSeconds} seconds...`); + await sleep(retryAfterSeconds * 1000) + continue; + } + } + throw new Error(`Failed bootstrapping ${envSpecifier}`); } } diff --git a/packages/@aws-cdk-testing/cli-integ/tests/tool-integrations/amplify.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/tool-integrations/amplify.integtest.ts index 138ebdc..c99788c 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/tool-integrations/amplify.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/tool-integrations/amplify.integtest.ts @@ -28,9 +28,12 @@ integTest('amplify integration', withToolContext(async (context) => { await shell.shell(['npm', 'create', '-y', 'amplify']); await shell.shell(['npx', 'ampx', 'configure', 'telemetry', 'disable']); + const awsCreds = context.aws.identityEnv(); + await shell.shell(['npx', 'ampx', 'sandbox', '--once'], { modEnv: { AWS_REGION: context.aws.region, + ...awsCreds }, }); try { @@ -42,6 +45,7 @@ integTest('amplify integration', withToolContext(async (context) => { await shell.shell(['npx', 'ampx', 'sandbox', 'delete', '--yes'], { modEnv: { AWS_REGION: context.aws.region, + ...awsCreds }, }); } diff --git a/yarn.lock b/yarn.lock index 01db66e..28b6a0f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3651,12 +3651,12 @@ case@^1.6.3: resolved "https://registry.yarnpkg.com/case/-/case-1.6.3.tgz#0a4386e3e9825351ca2e6216c60467ff5f1ea1c9" integrity sha512-mzDSXIPaFwVDvZAHqZ9VlbyF4yyXRuX6IvB06WvPYkqJVO24kX1PPhv9bfpKNFZyxYFmmgo03HUiD8iklmJYRQ== -cdklabs-projen-project-types@^0.2.4: - version "0.2.4" - resolved "https://registry.yarnpkg.com/cdklabs-projen-project-types/-/cdklabs-projen-project-types-0.2.4.tgz#a81ad12fcc82a79aa61f0f146a57be2a6c2ca69d" - integrity sha512-T6XKFJnwpHlwn0MKGRJ1y5nEHB03Rl+KFJaP1+6TltkCzkX4gvzzvXANKOGYvQpZexACpCgQzcUD5hrSY4Hu0Q== +cdklabs-projen-project-types@^0.2.13: + version "0.2.13" + resolved "https://registry.yarnpkg.com/cdklabs-projen-project-types/-/cdklabs-projen-project-types-0.2.13.tgz#fcd19b1c006f74a4802f927492e6a0b4872806d9" + integrity sha512-tRY+ewbmPYhQEo9d32rQ2inUNG6vkjaFb4wAaKsjvycwBCkHhlb6KXuHH1olri8D8dx/QwC90WUo5cp7pxbzKg== dependencies: - yaml "^2.7.0" + yaml "^2.7.1" chalk@^2.4.2: version "2.4.2" @@ -9312,11 +9312,16 @@ yaml@1: resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== -yaml@^2.2.2, yaml@^2.4.1, yaml@^2.7.0: +yaml@^2.2.2, yaml@^2.4.1: version "2.7.0" resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.7.0.tgz#aef9bb617a64c937a9a748803786ad8d3ffe1e98" integrity sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA== +yaml@^2.7.1: + version "2.7.1" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.7.1.tgz#44a247d1b88523855679ac7fa7cda6ed7e135cf6" + integrity sha512-10ULxpnOCQXxJvBgxsn9ptjq6uviG/htZKk9veJGhlqn3w/DxQ631zFF+nlQXLwmImeS5amR2dl2U8sg6U9jsQ== + yargs-parser@^20.2.2, yargs-parser@^20.2.3: version "20.2.9" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee"