Terraform 0.13 Compatibility (#285)

* Ignore terraform providers for docker build - This can get huge otherwise

* Use tfenv for Terraform version management in Dockerfile

* Allow adding source

* Test 0.12 and 0.13

* Update gitignore

* Make docker example work

* Cleanup obsolete modules

* Fix build

* Linter

* Update snapshots

* Drop schema from generated files

We were planning to do this anyway. It became necessary now, since Terraform 0.13
added at least one new field to the schema (`description_kind`). This caused our
snapshot tests to fail when running with different terraform versions

* Adjust integration tests

* adding new jsii image and the using custom terraform binary names

Co-authored-by: Anubhav Mishra <[email protected]>

.dockerignore

@@ -9,4 +9,5 @@
 
 **/cdktf.out
 **/.gen
-**/dist
+**/dist
+**/.terraform

.github/workflows/build.yml

@@ -4,6 +4,9 @@ on: [pull_request]
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        terraform: ["0.12.29", 0.13.0-rc1]
     container:
       image: hashicorp/jsii-terraform
 
@@ -16,12 +19,17 @@ jobs:
         run: |
           tools/align-version.sh
           yarn build
+        env:
+          TERRAFORM_BINARY_NAME: "terraform${{ matrix.terraform }}"
       - name: test
         run: |
           yarn test
+        env:
+          TERRAFORM_BINARY_NAME: "terraform${{ matrix.terraform }}"
       - name: create bundle
         run: yarn package
       - name: integration tests
         run: yarn integration
         env:
-          TERRAFORM_CLOUD_TOKEN: ${{ secrets.TERRAFORM_CLOUD_TOKEN }}
+          TERRAFORM_CLOUD_TOKEN: ${{ secrets.TERRAFORM_CLOUD_TOKEN }}
+          TERRAFORM_BINARY_NAME: "terraform${{ matrix.terraform }}"

.gitignore

@@ -16,4 +16,5 @@ tsconfig.json
 **/*.log
 **/coverage
 **/dist
-**/.terraform
+**/.terraform
+.vscode

Dockerfile

@@ -1,7 +1,13 @@
 FROM jsii/superchain
 
-COPY tools/install-terraform.sh /tmp/
+RUN yum install -y unzip && curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
 
-RUN yum install -y jq && /tmp/install-terraform.sh && curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
+ENV DEFAULT_TERRAFORM_VERSION=0.13.0-rc1
 
-RUN rm -f /tmp/install-terraform.sh && yum remove -y jq
+# Install Terraform
+RUN AVAILABLE_TERRAFORM_VERSIONS="0.12.29 ${DEFAULT_TERRAFORM_VERSION}" && \
+    for VERSION in ${AVAILABLE_TERRAFORM_VERSIONS}; do curl -LOk https://releases.hashicorp.com/terraform/${VERSION}/terraform_${VERSION}_linux_amd64.zip && \
+    mkdir -p /usr/local/bin/tf/versions/${VERSION} && \
+    unzip terraform_${VERSION}_linux_amd64.zip -d /usr/local/bin/tf/versions/${VERSION} && \
+    ln -s /usr/local/bin/tf/versions/${VERSION}/terraform /usr/local/bin/terraform${VERSION};rm terraform_${VERSION}_linux_amd64.zip;done && \
+    ln -s /usr/local/bin/tf/versions/${DEFAULT_TERRAFORM_VERSION}/terraform /usr/local/bin/terraform

examples/python/docker/cdktf.json

@@ -1,6 +1,8 @@
 {
   "language": "python",
   "app": "pipenv run ./main.py",
-  "terraformProviders": ["docker"],
+  "terraformProviders": [
+    "terraform-providers/docker@~> 2.0"
+  ],
   "codeMakerOutput": "imports"
-}
+}

examples/python/docker/main.py

@@ -1,15 +1,17 @@
 #!/usr/bin/python3 -tt
 from constructs import Construct
 from cdktf import App, TerraformStack
-from imports.docker import Image, Container
+from imports.docker import Image, Container, DockerProvider
 
 
 class MyStack(TerraformStack):
     def __init__(self, scope: Construct, ns: str):
         super().__init__(scope, ns)
-
+
+        DockerProvider(self, "provider")
+
         docker_image = Image(self, 'nginx-latest', name='nginx:latest', keep_locally=False)
-        
+
         Container(self, 'nginx-cdktf', name='nginx-python-cdktf',
                   image=docker_image.name, ports=[
                       {

examples/typescript/aws/cdktf.json

@@ -3,9 +3,5 @@
   "app": "npm run --silent compile && node main.js",
   "terraformProviders": [
     "aws@~> 2.0"
-  ],
-  "terraformModules": [
-    "terraform-aws-modules/eks/aws",
-    "terraform-aws-modules/vpc/aws"
   ]
 }

examples/typescript/docker/cdktf.json

@@ -2,6 +2,6 @@
   "language": "typescript",
   "app": "npm run --silent compile && node main.js",
   "terraformProviders": [
-    "docker"
+    "terraform-providers/docker@~> 2.0"
   ]
 }

examples/typescript/docker/main.ts

@@ -15,14 +15,16 @@ Steps:
 
 import { Construct } from 'constructs';
 import { App, TerraformStack } from 'cdktf';
-import { Container, Image } from './.gen/providers/docker';
+import { Container, Image, DockerProvider } from './.gen/providers/docker';
 
 class MyStack extends TerraformStack {
   public readonly dockerImage: Image
 
   constructor(scope: Construct, name: string) {
     super(scope, name);
 
+    new DockerProvider(this, 'provider', {})
+
     this.dockerImage = new Image(this, 'nginxImage', {
       name         : "nginx:latest",
       keepLocally : false
@@ -35,7 +37,7 @@ class MyStack extends TerraformStack {
         internal: 80,
         external: 8000
       }]
-    });    
+    });
 
   }
 }

package.json

@@ -9,7 +9,9 @@
     "watch": "lerna run --parallel  --stream  --scope cdktf* watch-preserve-output",
     "link-packages": "lerna exec --scope cdktf* yarn link",
     "integration": "test/run-against-dist test/test-all.sh",
-    "release-github": "tools/release-github.sh"
+    "release-github": "tools/release-github.sh",
+    "build-docker-jsii": "docker build -t hashicorp/jsii-terraform .",
+    "push-docker-jsii": "docker push hashicorp/jsii-terraform"
   },
   "workspaces": {
     "packages": [

packages/cdktf-cli/bin/cmds/ui/models/terraform.ts

@@ -1,6 +1,8 @@
 import * as path from 'path';
 import { exec } from '../../../../lib/util'
 
+const terraformBinaryName = process.env.TERRAFORM_BINARY_NAME || 'terraform'
+
 export enum PlannedResourceAction {
   CREATE = 'create',
   UPDATE = 'update',
@@ -81,7 +83,7 @@ export class Terraform  {
   }
 
   public async init(): Promise<void> {
-    await exec('terraform', ['init'], { cwd: this.workdir, env: process.env })
+    await exec(terraformBinaryName, ['init'], { cwd: this.workdir, env: process.env })
   }
 
   public async plan(destroy = false): Promise<TerraformPlan> {
@@ -90,29 +92,29 @@ export class Terraform  {
     if (destroy) {
       options.push('-destroy')
     }
-    await exec('terraform', options, { cwd: this.workdir, env: process.env });
-    const jsonPlan = await exec('terraform', ['show', '-json', planFile], { cwd: this.workdir, env: process.env });
+    await exec(terraformBinaryName, options, { cwd: this.workdir, env: process.env });
+    const jsonPlan = await exec(terraformBinaryName, ['show', '-json', planFile], { cwd: this.workdir, env: process.env });
     return new TerraformPlan(planFile, JSON.parse(jsonPlan));
   }
 
   public async deploy(planFile: string, stdout: (chunk: Buffer) => any): Promise<void> {
-    await exec('terraform', ['apply', '-auto-approve', ...this.stateFileOption, planFile], { cwd: this.workdir, env: process.env }, stdout);
+    await exec(terraformBinaryName, ['apply', '-auto-approve', ...this.stateFileOption, planFile], { cwd: this.workdir, env: process.env }, stdout);
   }
 
   public async destroy(stdout: (chunk: Buffer) => any): Promise<void> {
-    await exec('terraform', ['destroy', '-auto-approve', ...this.stateFileOption], { cwd: this.workdir, env: process.env }, stdout);
+    await exec(terraformBinaryName, ['destroy', '-auto-approve', ...this.stateFileOption], { cwd: this.workdir, env: process.env }, stdout);
   }
 
   public async version(): Promise<string> {
     try {
-      return await exec('terraform', ['-v'], { cwd: this.workdir, env: process.env });
+      return await exec(terraformBinaryName, ['-v'], { cwd: this.workdir, env: process.env });
     } catch {
       throw new Error("Terraform CLI not present - Please install a current version https://learn.hashicorp.com/terraform/getting-started/install.html")
     }
   }
 
   public async output(): Promise<{[key: string]: TerraformOutput}> {
-    const output = await exec('terraform', ['output', '-json', ...this.stateFileOption], { cwd: this.workdir, env: process.env });    return JSON.parse(output)
+    const output = await exec(terraformBinaryName, ['output', '-json', ...this.stateFileOption], { cwd: this.workdir, env: process.env });    return JSON.parse(output)
   }
 
   private get stateFileOption() {

packages/cdktf-cli/lib/get/generator/emitter/resource-emitter.ts

@@ -87,7 +87,8 @@ export class ResourceEmitter {
       this.code.open(`terraformGeneratorMetadata: {`);
         this.code.line(`providerName: '${resource.provider}',`);
         this.code.line(`providerVersionConstraint: '${resource.providerVersionConstraint}'`);
-      this.code.close(`}`);
+        this.code.close(`},`);
+      this.code.line(`terraformProviderSource: '${resource.terraformProviderSource}'`);
     this.code.close(`});`);
   }
 }

packages/cdktf-cli/lib/get/generator/models/resource-model.ts

@@ -22,6 +22,7 @@ export class ResourceModel {
   public baseName: string;
   public provider: string;
   public providerVersionConstraint?: string;
+  public terraformProviderSource?: string;
   public fileName: string;
   public filePath: string;
   public attributes: AttributeModel[];

packages/cdktf-cli/lib/get/generator/provider-generator.ts

@@ -4,12 +4,56 @@ import { ResourceModel } from "./models"
 import { ResourceParser } from './resource-parser'
 import { ResourceEmitter, StructEmitter } from './emitter'
 
+export class TerraformProviderConstraint {
+  public version: string;
+  public source?: string;
+  public name: string
+  public fqn:  string;
+
+  constructor(public cdktfConstraint: string) {
+    const [ fqn, version ] = cdktfConstraint.split('@');
+    const nameParts = fqn.split('/');
+    const name = nameParts.pop();
+    if (!name) { throw new Error(`Provider name should be properly set in ${cdktfConstraint}`) }
+
+    this.name = name;
+    this.source = nameParts.join('/');
+    this.version = version;
+    this.fqn = fqn
+  }
+
+  public isMatching(terraformSchemaName: string): boolean {
+    const elements = terraformSchemaName.split('/')
+
+    if (elements.length === 1) {
+      return this.name === terraformSchemaName
+    } else {
+      const [hostname, scope, provider] = elements
+
+      if (!hostname || !scope || !provider) {
+        throw new Error(`can't handle ${terraformSchemaName}`)
+      }
+
+      return this.name === provider;
+    }
+  }
+}
+interface ProviderData {
+  name: string;
+  source: string;
+  version: string;
+}
+
+export interface ProviderConstraints {
+  [fqn: string]: ProviderData;
+}
+
 export class TerraformGenerator {
   private resourceParser = new ResourceParser();
   private resourceEmitter:  ResourceEmitter;
   private structEmitter:  StructEmitter;
+  constructor(private readonly code: CodeMaker, schema: ProviderSchema, private providerConstraints?: TerraformProviderConstraint[]) {
 
-  constructor(private readonly code: CodeMaker, schema: ProviderSchema, private providerConstraints?: { [name: string]: string }) {
     this.code.indentation = 2;
     this.resourceEmitter = new ResourceEmitter(this.code)
     this.structEmitter = new StructEmitter(this.code)
@@ -19,16 +63,19 @@ export class TerraformGenerator {
       return;
     }
 
-    for (const [name, provider] of Object.entries(schema.provider_schemas)) {
-      this.emitProvider(name, provider);
+    for (const [fqpn, provider] of Object.entries(schema.provider_schemas)) {
+      this.emitProvider(fqpn, provider);
     }
   }
 
   public async save(outdir: string) {
     await this.code.save(outdir);
   }
 
-  private emitProvider(name: string, provider: Provider) {
+  private emitProvider(fqpn: string, provider: Provider) {
+    const name = fqpn.split('/').pop()
+    if (!name) { throw new Error(`can't handle ${fqpn}`) }
+
     const files: string[] = []
     for (const [type, resource] of Object.entries(provider.resource_schemas)) {
       files.push(this.emitResourceFile(this.resourceParser.parse(name, type, resource, 'resource')));
@@ -41,7 +88,13 @@ export class TerraformGenerator {
     if (provider.provider) {
       const providerResource = this.resourceParser.parse(name, `provider`, provider.provider, 'provider')
       if (this.providerConstraints) {
-        providerResource.providerVersionConstraint = this.providerConstraints[name]
+        const constraint = this.providerConstraints.find((p) => (p.isMatching(fqpn)))
+        if (!constraint) {
+          console.log({foo: this.providerConstraints, fqpn})
+          throw new Error(`can't handle ${fqpn}`)
+        }
+        providerResource.providerVersionConstraint = constraint.version;
+        providerResource.terraformProviderSource = constraint.fqn;
       }
       files.push(this.emitResourceFile(providerResource));
     }
@@ -75,9 +128,6 @@ export class TerraformGenerator {
     this.code.line(`// ${resource.linkToDocs}`);
     this.code.line(`// generated from terraform resource schema`);
     this.code.line();
-    this.code.line('/*');
-    this.code.line(resource.schemaAsJson);
-    this.code.line('*/');
     resource.importStatements.forEach(statement => this.code.line(statement))
     this.code.line();
     this.code.line('// Configuration');

packages/cdktf-cli/lib/get/generator/provider-schema.ts

@@ -56,18 +56,25 @@ export interface Block {
 }
 
 export async function readSchema(providers: string[]): Promise<ProviderSchema> {
-  const provider: { [name: string]: { version?: string } } = { };
+  const provider: { [name: string]: {} } = {};
+  const requiredProviders: { [name: string]: { source?: string;  version?: string } } = { };
+
   for (const p of providers) {
-    const [ name, version ] = p.split('@');
-    provider[name] = { version };
+    const [ fqname, version ] = p.split('@');
+    const name = fqname.split('/').pop()
+    if (!name) { throw new Error(`Provider name should be properly set in ${p}`) }
+
+    provider[name] = {};
+    requiredProviders[name] = { version, source: fqname };
   }
   let schema = '';
   const workDir = process.cwd()
 
   await withTempDir('fetchSchema', async () => {
     const outdir = process.cwd();
     const filePath = path.join(outdir, 'providers.tf.json');
-    await writeFile(filePath, JSON.stringify({ provider }));
+    // eslint-disable-next-line @typescript-eslint/camelcase
+    await writeFile(filePath, JSON.stringify({ provider, terraform: { required_providers: requiredProviders }}));
 
     const env = process.env['TF_PLUGIN_CACHE_DIR'] ? process.env : Object.assign({}, process.env, { 'TF_PLUGIN_CACHE_DIR': await cacheDir(workDir) })
 

packages/cdktf-cli/lib/get/providers.ts

@@ -1,5 +1,5 @@
 // generates constructs from terraform providers schema
-import { TerraformGenerator } from './generator/provider-generator';
+import { TerraformGenerator, TerraformProviderConstraint } from './generator/provider-generator';
 import { ProviderSchema, readSchema } from './generator/provider-schema';
 import { CodeMaker } from 'codemaker';
 import { GetBase } from './base'
@@ -19,11 +19,10 @@ export class GetProvider extends GetBase {
     return `providers/${name}/index`;
   }
 
-  private async parseProviders(providers: string[]): Promise<{ [name: string]: string }> {
-    const provider: { [name: string]: string } = { };
+  private async parseProviders(providers: string[]): Promise<TerraformProviderConstraint[]> {
+    const provider: TerraformProviderConstraint[] = [];
     for (const p of providers) {
-      const [ name, version ] = p.split('@');
-      provider[name] = version;
+      provider.push(new TerraformProviderConstraint(p))
     }
     return provider;
   }

packages/cdktf-cli/package.json

@@ -10,6 +10,7 @@
     "watch": "tsc -w",
     "watch-preserve-output": "tsc -w --preserveWatchOutput",
     "lint": "eslint . --ext .ts",
+    "lint:fix": "eslint . --ext .ts --fix",
     "test": "yarn lint && jest",
     "jest-watch": "jest --watch",
     "package": "./package.sh"