Merge pull request #344 from hashicorp/no-symlink

Drop custom symlink for generic plugin cache setting

.github/workflows/examples.yml

@@ -9,10 +9,9 @@ jobs:
         terraform: ["0.12.29", "0.13.0"]
     container:
       image: hashicorp/jsii-terraform
-
     env:
+      TF_PLUGIN_CACHE_DIR: "/root/.terraform.d/plugin-cache"
       CHECKPOINT_DISABLE: "1"
-
     steps:
       - uses: actions/checkout@v2
       - name: installing dependencies
@@ -25,6 +24,8 @@ jobs:
           TERRAFORM_BINARY_NAME: "terraform${{ matrix.terraform }}"
       - name: create bundle
         run: yarn package
+      - name: bootstrap plugin cache
+        run: yum install -y jq && yarn bootstrap-plugin-cache
       - name: examples integration tests
         run: yarn examples:integration
         env:

.gitignore

@@ -18,4 +18,5 @@ tsconfig.json
 **/coverage
 **/dist
 **/.terraform
-.vscode
+.vscode
+bootstrap.json

Dockerfile

@@ -1,8 +1,9 @@
 FROM jsii/superchain
 
-RUN yum install -y unzip && curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
+RUN yum install -y unzip jq && curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
 
 ENV DEFAULT_TERRAFORM_VERSION=0.13.0
+ENV TF_PLUGIN_CACHE_DIR="/root/.terraform.d/plugin-cache"
 
 # Install Terraform
 RUN AVAILABLE_TERRAFORM_VERSIONS="0.12.29 0.13.0-rc1 ${DEFAULT_TERRAFORM_VERSION}" && \

docs/working-with-cdk-for-terraform/using-providers-and-modules.md

@@ -206,3 +206,13 @@ cdktf synth --json
 }
 
 ```
+
+## Provider Caching
+
+When using the `cdktf` cli commands, it'll automatically set the process env `TF_PLUGIN_CACHE_DIR` to `$HOME/.terraform.d/plugin-cache` if it isn't set to something else. This will avoid re-downlodading the providers between the different `cdktf` commands. See the [Terraform](https://www.terraform.io/docs/commands/cli-config.html#provider-plugin-cache) docs for more information.
+
+`cdktf get` works in a temporary directory, hence all downloaded providers would be lost without caching. For the deployment related commands `diff` / `deploy` / `destroy`, the working directory is usually `cdktf.out` and is treated as throwaway folder. While not common, it's totally reasonable to remove the `cdktf.out` folder and synthesize again. In that case, caching will help as well.
+
+Last but not least, when approaching multiple stacks wihtin oen application (not yet implemented), provider caching is a basic prerequisite.
+
+This behaviour can be disabled by setting `CDKTF_DISABLE_PLUGIN_CACHE_ENV` to non null value, e.g. `CDKTF_DISABLE_PLUGIN_CACHE_ENV=1`. This might be desired, when a different cache directory is configured via a `.terraformrc` configuration file.

package.json

@@ -6,6 +6,7 @@
     "build": "lerna run --scope cdktf* build",
     "package": "lerna run package && tools/collect-dist.sh",
     "package-windows": "lerna run package && tools\\collect-dist.bat",
+    "bootstrap-plugin-cache": "./test/run-against-dist ./tools/bootstrap-plugin-cache.sh",
     "examples:reinstall": "lerna run --parallel --scope @examples/* reinstall",
     "examples:build": "lerna run --parallel --scope @examples/* build",
     "examples:synth": "lerna run --parallel --scope @examples/* synth",

packages/cdktf-cli/bin/cdktf.ts

@@ -2,6 +2,21 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'production'
 
 import * as yargs from 'yargs';
 import * as semver from 'semver';
+import * as path from 'path';
+import * as os from 'os';
+import * as fs from 'fs-extra';
+
+const ensurePluginCache = (): string => {
+  const pluginCachePath = process.env.TF_PLUGIN_CACHE_DIR || path.join(os.homedir(), '.terraform.d', 'plugin-cache')
+  if (!fs.existsSync(pluginCachePath)) {
+    fs.mkdirpSync(pluginCachePath)
+  }
+  return pluginCachePath;
+}
+
+if (!process.env.CDKTF_DISABLE_PLUGIN_CACHE_ENV) {
+  process.env.TF_PLUGIN_CACHE_DIR = ensurePluginCache()
+}
 
 if (semver.lt(process.version, '10.12.0')) { console.error("Need at least Node v10.12 to run") ; process.exit(1) }
 
@@ -15,6 +30,7 @@ const args = yargs
   .help()
   .alias('h', 'help')
   .option('disable-logging', { type: 'boolean', default: true, required: false, desc: 'Dont write log files. Supported using the env CDKTF_DISABLE_LOGGING.'})
+  .option('disable-plugin-cache-env', { type: 'boolean', default: false, required: false, desc: 'Dont set TF_PLUGIN_CACHE_DIR automatically. This is useful when the plugin cache is configured differently. Supported using the env CDKTF_DISABLE_PLUGIN_CACHE_ENV.'})
   .option('log-level', { type: 'string', required: false, desc: 'Which log level should be written. Only supported via setting the env CDKTF_LOG_LEVEL'})
   .argv;
 

packages/cdktf-cli/lib/get/generator/provider-schema.ts

@@ -4,7 +4,6 @@ import { promisify } from 'util';
 import { exec, withTempDir } from '../../util';
 
 const writeFile = promisify(fs.writeFile);
-const mkdirp = promisify(fs.mkdirp);
 
 export interface ProviderSchema {
   format_version: '1.0';
@@ -68,28 +67,19 @@ export async function readSchema(providers: string[]): Promise<ProviderSchema> {
     requiredProviders[name] = { version, source: fqname };
   }
   let schema = '';
-  const workDir = process.cwd()
 
   await withTempDir('fetchSchema', async () => {
     const outdir = process.cwd();
     const filePath = path.join(outdir, 'providers.tf.json');
     // eslint-disable-next-line @typescript-eslint/camelcase
     await writeFile(filePath, JSON.stringify({ provider, terraform: { required_providers: requiredProviders }}));
 
-    const env = process.env['TF_PLUGIN_CACHE_DIR'] ? process.env : Object.assign({}, process.env, { 'TF_PLUGIN_CACHE_DIR': await cacheDir(workDir) })
-
     // todo: when implementing logging, we need to make sure we can show the terraform init
     // output if the log level is set to debug
-    await exec('terraform', [ 'init' ], { cwd: outdir, env });
-    schema = await exec('terraform', ['providers', 'schema', '-json'], { cwd: outdir, env });
+    await exec('terraform', [ 'init' ], { cwd: outdir });
+    schema = await exec('terraform', ['providers', 'schema', '-json'], { cwd: outdir });
     fs.unlinkSync(filePath)
   })
 
   return JSON.parse(schema);
 }
-
-async function cacheDir(workDir: string) {
-  const cacheDir = path.join(workDir, '.terraform/plugins');
-  await mkdirp(cacheDir);
-  return cacheDir
-}

packages/cdktf/lib/terraform-stack.ts

@@ -120,13 +120,5 @@ export class TerraformStack extends Construct {
   protected onSynthesize(session: ISynthesisSession) {
     const resourceOutput = path.join(session.outdir, this.artifactFile);
     fs.writeFileSync(resourceOutput, JSON.stringify(this.toTerraform(), undefined, 2));
-    this.linkDotTerraform(session.outdir)
-  }
-
-  private linkDotTerraform(outdir: string): void {
-    const dirName = '.terraform';
-    const link = path.join(path.resolve(outdir), dirName);
-    const target = path.join(process.cwd(), dirName);
-    if (!fs.existsSync(link)) fs.symlinkSync(target, link);
   }
 }

tools/bootstrap-plugin-cache.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+set -euo pipefail
+
+# Disable spinner even when we have a TTY
+export CI='1'
+
+find ./examples -type f -name '*cdktf.json' |  tr '\n' " " | xargs jq -s "map(.terraformProviders) | add | unique | { \"language\": \"typescript\", \"terraformProviders\": . }" > bootstrap.json
+
+scriptdir=$(cd $(dirname $0) && pwd)
+
+cd $(mktemp -d)
+mkdir test && cd test
+
+cdktf init --template typescript-minimal --project-name="typescript-bootstrap" --project-description="typescript bootstrap plugin cache" --local
+
+# add null provider
+mv ${scriptdir}/../bootstrap.json ./cdktf.json
+cdktf get
+
+echo "DONE"
+

yarn.lock

@@ -1577,6 +1577,11 @@
   resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-1.0.1.tgz#0a851d3bd96498fa25c33ab7278ed3bd65f06c3e"
   integrity sha512-l42BggppR6zLmpfU6fq9HEa2oGPEI8yrSPL3GITjfRInppYFahObbIQOQK3UGxEnyQpltZLaPe75046NOZQikw==
 
+"@types/uuid@^8.3.0":
+  version "8.3.0"
+  resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-8.3.0.tgz#215c231dff736d5ba92410e6d602050cce7e273f"
+  integrity sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==
+
 "@types/yargs-parser@*":
   version "15.0.0"
   resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-15.0.0.tgz#cb3f9f741869e20cce330ffbeb9271590483882d"
@@ -8197,6 +8202,11 @@ uuid@^3.0.1, uuid@^3.3.2:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
   integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
 
+uuid@^8.3.0:
+  version "8.3.0"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.0.tgz#ab738085ca22dc9a8c92725e459b1d507df5d6ea"
+  integrity sha512-fX6Z5o4m6XsXBdli9g7DtWgAx+osMsRRZFKma1mIUsLCz6vRvv+pz5VNbyu9UEDzpMWulZfvpgb/cmDXVulYFQ==
+
 v8-compile-cache@^2.0.3:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz#54bc3cdd43317bca91e35dcaf305b1a7237de745"