From 33909b4b003dce8f933ba47d553e8c9b17ac7acb Mon Sep 17 00:00:00 2001 From: Sebastian Korfmann Date: Mon, 24 Aug 2020 20:33:50 +0200 Subject: [PATCH 1/8] Drop custom symlink for generic plugin cache setting Since the current symlink approach for caching poviders has multiple drawbacks, this PR switches to the general plugin cache dir supported by Terraform directly. https://www.terraform.io/docs/commands/cli-config.html#provider-plugin-cache Fixes #189 Fixes #222 --- packages/cdktf-cli/bin/cdktf.ts | 12 ++++++++++++ .../cdktf-cli/lib/get/generator/provider-schema.ts | 14 ++------------ packages/cdktf/lib/terraform-stack.ts | 8 -------- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/packages/cdktf-cli/bin/cdktf.ts b/packages/cdktf-cli/bin/cdktf.ts index 67a161d65a..a321075add 100644 --- a/packages/cdktf-cli/bin/cdktf.ts +++ b/packages/cdktf-cli/bin/cdktf.ts @@ -2,7 +2,19 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'production' import * as yargs from 'yargs'; import * as semver from 'semver'; +import * as path from 'path'; +import * as os from 'os'; +import * as fs from 'fs-extra'; +const ensurePluginCache = (): string => { + const pluginCachePath = process.env.TF_PLUGIN_CACHE_DIR || path.join(os.homedir(), '.terraform.d', 'plugin-cache') + if (!fs.existsSync(pluginCachePath)) { + fs.mkdirpSync(pluginCachePath) + } + return pluginCachePath; +} + +process.env.TF_PLUGIN_CACHE_DIR = ensurePluginCache() if (semver.lt(process.version, '10.12.0')) { console.error("Need at least Node v10.12 to run") ; process.exit(1) } const args = yargs diff --git a/packages/cdktf-cli/lib/get/generator/provider-schema.ts b/packages/cdktf-cli/lib/get/generator/provider-schema.ts index c657c2dfd3..62b43e82da 100644 --- a/packages/cdktf-cli/lib/get/generator/provider-schema.ts +++ b/packages/cdktf-cli/lib/get/generator/provider-schema.ts @@ -4,7 +4,6 @@ import { promisify } from 'util'; import { exec, withTempDir } from '../../util'; const writeFile = promisify(fs.writeFile); -const mkdirp = promisify(fs.mkdirp); export interface ProviderSchema { format_version: '1.0'; @@ -68,7 +67,6 @@ export async function readSchema(providers: string[]): Promise { requiredProviders[name] = { version, source: fqname }; } let schema = ''; - const workDir = process.cwd() await withTempDir('fetchSchema', async () => { const outdir = process.cwd(); @@ -76,20 +74,12 @@ export async function readSchema(providers: string[]): Promise { // eslint-disable-next-line @typescript-eslint/camelcase await writeFile(filePath, JSON.stringify({ provider, terraform: { required_providers: requiredProviders }})); - const env = process.env['TF_PLUGIN_CACHE_DIR'] ? process.env : Object.assign({}, process.env, { 'TF_PLUGIN_CACHE_DIR': await cacheDir(workDir) }) - // todo: when implementing logging, we need to make sure we can show the terraform init // output if the log level is set to debug - await exec('terraform', [ 'init' ], { cwd: outdir, env }); - schema = await exec('terraform', ['providers', 'schema', '-json'], { cwd: outdir, env }); + await exec('terraform', [ 'init' ], { cwd: outdir }); + schema = await exec('terraform', ['providers', 'schema', '-json'], { cwd: outdir }); fs.unlinkSync(filePath) }) return JSON.parse(schema); } - -async function cacheDir(workDir: string) { - const cacheDir = path.join(workDir, '.terraform/plugins'); - await mkdirp(cacheDir); - return cacheDir -} diff --git a/packages/cdktf/lib/terraform-stack.ts b/packages/cdktf/lib/terraform-stack.ts index 2efdfe79c7..d97fe75600 100644 --- a/packages/cdktf/lib/terraform-stack.ts +++ b/packages/cdktf/lib/terraform-stack.ts @@ -120,13 +120,5 @@ export class TerraformStack extends Construct { protected onSynthesize(session: ISynthesisSession) { const resourceOutput = path.join(session.outdir, this.artifactFile); fs.writeFileSync(resourceOutput, JSON.stringify(this.toTerraform(), undefined, 2)); - this.linkDotTerraform(session.outdir) - } - - private linkDotTerraform(outdir: string): void { - const dirName = '.terraform'; - const link = path.join(path.resolve(outdir), dirName); - const target = path.join(process.cwd(), dirName); - if (!fs.existsSync(link)) fs.symlinkSync(target, link); } } From 7b292a86361d19580e367cfeeb294d7dadd3b69f Mon Sep 17 00:00:00 2001 From: Sebastian Korfmann Date: Tue, 25 Aug 2020 10:32:48 +0200 Subject: [PATCH 2/8] Bootstrap plugin cache --- .github/workflows/examples.yml | 5 ++++- .gitignore | 3 ++- Dockerfile | 1 + package.json | 1 + tools/bootstrap-plugin-cache.sh | 21 +++++++++++++++++++++ 5 files changed, 29 insertions(+), 2 deletions(-) create mode 100755 tools/bootstrap-plugin-cache.sh diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index ac5e7a5259..88ee84522f 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -9,7 +9,8 @@ jobs: terraform: ["0.12.29", "0.13.0"] container: image: hashicorp/jsii-terraform - + env: + TF_PLUGIN_CACHE_DIR: "/root/.terraform.d/plugin-cache" steps: - uses: actions/checkout@v2 - name: installing dependencies @@ -22,6 +23,8 @@ jobs: TERRAFORM_BINARY_NAME: "terraform${{ matrix.terraform }}" - name: create bundle run: yarn package + - name: bootstrap plugin cache + run: yarn bootstrap-plugin-cache - name: examples integration tests run: yarn examples:integration env: diff --git a/.gitignore b/.gitignore index edd6f8b3ac..e7fa8644e7 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ tsconfig.json **/coverage **/dist **/.terraform -.vscode \ No newline at end of file +.vscode +bootstrap.json \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f176477338..12f5bfc1f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,7 @@ FROM jsii/superchain RUN yum install -y unzip && curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python ENV DEFAULT_TERRAFORM_VERSION=0.13.0 +ENV TF_PLUGIN_CACHE_DIR="/root/.terraform.d/plugin-cache" # Install Terraform RUN AVAILABLE_TERRAFORM_VERSIONS="0.12.29 0.13.0-rc1 ${DEFAULT_TERRAFORM_VERSION}" && \ diff --git a/package.json b/package.json index ff5d9140b3..0b6f3237c5 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "build": "lerna run --scope cdktf* build", "package": "lerna run package && tools/collect-dist.sh", "package-windows": "lerna run package && tools\\collect-dist.bat", + "bootstrap-plugin-cache": "./test/run-against-dist ./tools/bootstrap-plugin-cache.sh", "examples:reinstall": "lerna run --parallel --scope @examples/* reinstall", "examples:build": "lerna run --parallel --scope @examples/* build", "examples:synth": "lerna run --parallel --scope @examples/* synth", diff --git a/tools/bootstrap-plugin-cache.sh b/tools/bootstrap-plugin-cache.sh new file mode 100755 index 0000000000..f6c97027b9 --- /dev/null +++ b/tools/bootstrap-plugin-cache.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -euo pipefail + +# Disable spinner even when we have a TTY +export CI='1' + +find ./examples -type f -name '*cdktf.json' | tr '\n' " " | xargs jq -s "map(.terraformProviders) | add | unique | { \"language\": \"typescript\", \"terraformProviders\": . }" > bootstrap.json + +scriptdir=$(cd $(dirname $0) && pwd) + +cd $(mktemp -d) +mkdir test && cd test + +cdktf init --template typescript-minimal --project-name="typescript-bootstrap" --project-description="typescript bootstrap plugin cache" --local + +# add null provider +mv ${scriptdir}/../bootstrap.json ./cdktf.json +cdktf get + +echo "DONE" + From f2389a1cd626a9479274034c684f88cce2d5cd5a Mon Sep 17 00:00:00 2001 From: Sebastian Korfmann Date: Tue, 25 Aug 2020 10:39:59 +0200 Subject: [PATCH 3/8] Install jq --- .github/workflows/examples.yml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 88ee84522f..b568176e28 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -24,7 +24,7 @@ jobs: - name: create bundle run: yarn package - name: bootstrap plugin cache - run: yarn bootstrap-plugin-cache + run: yum install -y jq && yarn bootstrap-plugin-cache - name: examples integration tests run: yarn examples:integration env: diff --git a/Dockerfile b/Dockerfile index 12f5bfc1f8..59771c9d45 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM jsii/superchain -RUN yum install -y unzip && curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python +RUN yum install -y unzip jq && curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python ENV DEFAULT_TERRAFORM_VERSION=0.13.0 ENV TF_PLUGIN_CACHE_DIR="/root/.terraform.d/plugin-cache" From 10f024be26af36c6aed2a1ad9e49134257dca04b Mon Sep 17 00:00:00 2001 From: Sebastian Korfmann Date: Tue, 25 Aug 2020 11:12:59 +0200 Subject: [PATCH 4/8] Allow to disable the plugin cache This is primarily useful when configuring the plugin cache via terraform.rc or similar means --- packages/cdktf-cli/bin/cdktf.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cdktf-cli/bin/cdktf.ts b/packages/cdktf-cli/bin/cdktf.ts index a321075add..9eddbf78f4 100644 --- a/packages/cdktf-cli/bin/cdktf.ts +++ b/packages/cdktf-cli/bin/cdktf.ts @@ -14,7 +14,10 @@ const ensurePluginCache = (): string => { return pluginCachePath; } -process.env.TF_PLUGIN_CACHE_DIR = ensurePluginCache() +if (!process.env.DISABLE_PLUGIN_CACHE_ENV) { + process.env.TF_PLUGIN_CACHE_DIR = ensurePluginCache() +} + if (semver.lt(process.version, '10.12.0')) { console.error("Need at least Node v10.12 to run") ; process.exit(1) } const args = yargs @@ -27,6 +30,7 @@ const args = yargs .help() .alias('h', 'help') .option('disable-logging', { type: 'boolean', default: true, required: false, desc: 'Dont write log files. Supported using the env CDKTF_DISABLE_LOGGING.'}) + .option('disable-plugin-cache-env', { type: 'boolean', default: false, required: false, desc: 'Dont set TF_PLUGIN_CACHE_DIR automatically. This is useful when the plugin cache is configured differently. Supported using the env DISABLE_PLUGIN_CACHE_ENV.'}) .option('log-level', { type: 'string', required: false, desc: 'Which log level should be written. Only supported via setting the env CDKTF_LOG_LEVEL'}) .argv; From f83034cc054e210870358e1f6599917bdca7107a Mon Sep 17 00:00:00 2001 From: Sebastian Korfmann Date: Thu, 27 Aug 2020 10:39:49 +0200 Subject: [PATCH 5/8] Update packages/cdktf-cli/bin/cdktf.ts Co-authored-by: Anubhav Mishra --- packages/cdktf-cli/bin/cdktf.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cdktf-cli/bin/cdktf.ts b/packages/cdktf-cli/bin/cdktf.ts index 9eddbf78f4..d79e11f603 100644 --- a/packages/cdktf-cli/bin/cdktf.ts +++ b/packages/cdktf-cli/bin/cdktf.ts @@ -30,7 +30,7 @@ const args = yargs .help() .alias('h', 'help') .option('disable-logging', { type: 'boolean', default: true, required: false, desc: 'Dont write log files. Supported using the env CDKTF_DISABLE_LOGGING.'}) - .option('disable-plugin-cache-env', { type: 'boolean', default: false, required: false, desc: 'Dont set TF_PLUGIN_CACHE_DIR automatically. This is useful when the plugin cache is configured differently. Supported using the env DISABLE_PLUGIN_CACHE_ENV.'}) + .option('disable-plugin-cache-env', { type: 'boolean', default: false, required: false, desc: 'Dont set TF_PLUGIN_CACHE_DIR automatically. This is useful when the plugin cache is configured differently. Supported using the env CDKTF_DISABLE_PLUGIN_CACHE_ENV.'}) .option('log-level', { type: 'string', required: false, desc: 'Which log level should be written. Only supported via setting the env CDKTF_LOG_LEVEL'}) .argv; From 616f7158e3a8dca7da9a5993979c18c53e7bd906 Mon Sep 17 00:00:00 2001 From: Sebastian Korfmann Date: Thu, 27 Aug 2020 10:39:55 +0200 Subject: [PATCH 6/8] Update packages/cdktf-cli/bin/cdktf.ts Co-authored-by: Anubhav Mishra --- packages/cdktf-cli/bin/cdktf.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cdktf-cli/bin/cdktf.ts b/packages/cdktf-cli/bin/cdktf.ts index d79e11f603..346f34146c 100644 --- a/packages/cdktf-cli/bin/cdktf.ts +++ b/packages/cdktf-cli/bin/cdktf.ts @@ -14,7 +14,7 @@ const ensurePluginCache = (): string => { return pluginCachePath; } -if (!process.env.DISABLE_PLUGIN_CACHE_ENV) { +if (!process.env.CDKTF_DISABLE_PLUGIN_CACHE_ENV) { process.env.TF_PLUGIN_CACHE_DIR = ensurePluginCache() } From a64a76d2f3a5ddec6ae59ab0f184d77a05ea5683 Mon Sep 17 00:00:00 2001 From: Sebastian Korfmann Date: Mon, 31 Aug 2020 16:05:26 +0200 Subject: [PATCH 7/8] Update yarn lock --- yarn.lock | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/yarn.lock b/yarn.lock index 0ab3f3346a..1a73d5f685 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1577,6 +1577,11 @@ resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-1.0.1.tgz#0a851d3bd96498fa25c33ab7278ed3bd65f06c3e" integrity sha512-l42BggppR6zLmpfU6fq9HEa2oGPEI8yrSPL3GITjfRInppYFahObbIQOQK3UGxEnyQpltZLaPe75046NOZQikw== +"@types/uuid@^8.3.0": + version "8.3.0" + resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-8.3.0.tgz#215c231dff736d5ba92410e6d602050cce7e273f" + integrity sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ== + "@types/yargs-parser@*": version "15.0.0" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-15.0.0.tgz#cb3f9f741869e20cce330ffbeb9271590483882d" @@ -8197,6 +8202,11 @@ uuid@^3.0.1, uuid@^3.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== +uuid@^8.3.0: + version "8.3.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.0.tgz#ab738085ca22dc9a8c92725e459b1d507df5d6ea" + integrity sha512-fX6Z5o4m6XsXBdli9g7DtWgAx+osMsRRZFKma1mIUsLCz6vRvv+pz5VNbyu9UEDzpMWulZfvpgb/cmDXVulYFQ== + v8-compile-cache@^2.0.3: version "2.1.1" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz#54bc3cdd43317bca91e35dcaf305b1a7237de745" From a78071d060e9d786fd2259b099bfa347c6cf02c3 Mon Sep 17 00:00:00 2001 From: Sebastian Korfmann Date: Mon, 31 Aug 2020 16:19:51 +0200 Subject: [PATCH 8/8] Add docs about provider caching --- .../using-providers-and-modules.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/working-with-cdk-for-terraform/using-providers-and-modules.md b/docs/working-with-cdk-for-terraform/using-providers-and-modules.md index dbf242766b..7faf9711fa 100644 --- a/docs/working-with-cdk-for-terraform/using-providers-and-modules.md +++ b/docs/working-with-cdk-for-terraform/using-providers-and-modules.md @@ -206,3 +206,13 @@ cdktf synth --json } ``` + +## Provider Caching + +When using the `cdktf` cli commands, it'll automatically set the process env `TF_PLUGIN_CACHE_DIR` to `$HOME/.terraform.d/plugin-cache` if it isn't set to something else. This will avoid re-downlodading the providers between the different `cdktf` commands. See the [Terraform](https://www.terraform.io/docs/commands/cli-config.html#provider-plugin-cache) docs for more information. + +`cdktf get` works in a temporary directory, hence all downloaded providers would be lost without caching. For the deployment related commands `diff` / `deploy` / `destroy`, the working directory is usually `cdktf.out` and is treated as throwaway folder. While not common, it's totally reasonable to remove the `cdktf.out` folder and synthesize again. In that case, caching will help as well. + +Last but not least, when approaching multiple stacks wihtin oen application (not yet implemented), provider caching is a basic prerequisite. + +This behaviour can be disabled by setting `CDKTF_DISABLE_PLUGIN_CACHE_ENV` to non null value, e.g. `CDKTF_DISABLE_PLUGIN_CACHE_ENV=1`. This might be desired, when a different cache directory is configured via a `.terraformrc` configuration file. \ No newline at end of file