diff --git a/.container-structure-test-config.yaml b/.container-structure-test-config.yaml new file mode 100644 index 000000000..fae1111eb --- /dev/null +++ b/.container-structure-test-config.yaml @@ -0,0 +1,44 @@ +schemaVersion: '2.0.0' +commandTests: + - name: "pre-commit" + command: "pre-commit" + args: ["-V"] + expectedOutput: ["^pre-commit ([0-9]+\\.){2}[0-9]+\\n$"] + - name: "terraform" + command: "terraform" + args: ["-version"] + expectedOutput: ["^Terraform v([0-9]+\\.){2}[0-9]+\\non linux_amd64\\n$"] + - name: "checkov" + command: "checkov" + args: ["--version"] + expectedOutput: ["^([0-9]+\\.){2}[0-9]+\\n$"] + - name: "infracost" + command: "infracost" + args: ["--version"] + expectedOutput: ["^Infracost v([0-9]+\\.){2}[0-9]+\\n$"] + - name: "terraform-docs" + command: "terraform-docs" + args: ["--version"] + expectedOutput: ["^terraform-docs version v([0-9]+\\.){2}[0-9]+ [a-z0-9]+ linux/amd64\\n$"] + - name: "terragrunt" + command: "terragrunt" + args: ["--version"] + expectedOutput: ["^terragrunt version v([0-9]+\\.){2}[0-9]+\\n$"] + - name: "terrascan" + command: "terrascan" + args: [ "version" ] + expectedOutput: [ "^version: v([0-9]+\\.){2}[0-9]+\\n$" ] + - name: "tflint" + command: "tflint" + args: [ "--version" ] + expectedOutput: [ "TFLint version ([0-9]+\\.){2}[0-9]+\\n$" ] + - name: "tfsec" + command: "tfsec" + args: [ "--version" ] + expectedOutput: [ "([0-9]+\\.){2}[0-9]+\\n$" ] +fileExistenceTests: + - name: 'terrascan init' + path: '/root/.terrascan/pkg/policies/opa/rego/github/github_repository/privateRepoEnabled.rego' + shouldExist: true + uid: 0 + gid: 0 diff --git a/.dive-ci.yaml b/.dive-ci.yaml new file mode 100644 index 000000000..008c19b76 --- /dev/null +++ b/.dive-ci.yaml @@ -0,0 +1,13 @@ +rules: + # If the efficiency is measured below X%, mark as failed. + # Expressed as a ratio between 0-1. + lowestEfficiency: 0.99 + + # If the amount of wasted space is at least X or larger than X, mark as failed. + # Expressed in B, KB, MB, and GB. + highestWastedBytes: 1MB + + # If the amount of wasted space makes up for X% or more of the image, mark as failed. + # Note: the base image layer is NOT included in the total image size. + # Expressed as a ratio between 0-1; fails if the threshold is met or crossed. + highestUserWastedPercent: 0.5 diff --git a/.dockerignore b/.dockerignore index 50c8ea340..2b9396ae0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,7 @@ -* -!.dockerignore -!Dockerfile +Dockerfile +.dockerignore +.editorconfig +.pre-commit-config.yaml +.github +.gitignore +.hadolint.yaml diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index c096a7695..be9c603ba 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -12,8 +12,13 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + - uses: hadolint/hadolint-action@v1.6.0 + with: + dockerfile: ./Dockerfile - name: Set up QEMU uses: docker/setup-qemu-action@v1 + with: + platforms: arm64 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - name: Login to GitHub Container Registry @@ -31,7 +36,27 @@ jobs: context: . build-args: | INSTALL_ALL=true - platforms: linux/amd64 + load: true + tags: | + ghcr.io/${{ github.repository }}:${{ env.IMAGE_TAG }} + - name: run structure tests + uses: plexsystems/container-structure-test-action@v0.1.0 + with: + image: ghcr.io/${{ github.repository }}:${{ env.IMAGE_TAG }} + config: ${{ github.workspace }}/.container-structure-test-config.yaml + - name: Dive + uses: yuichielectric/dive-action@0.0.4 + with: + image: ghcr.io/${{ github.repository }}:${{ env.IMAGE_TAG }} + config-file: ${{ github.workspace }}/.dive-ci.yml + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build and Push + uses: docker/build-push-action@v2 + with: + context: . + build-args: | + INSTALL_ALL=true + platforms: linux/amd64,linux/arm64 push: true tags: | ghcr.io/${{ github.repository }}:${{ env.IMAGE_TAG }} diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 6f2ec84ff..5f50f62ba 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -25,6 +25,13 @@ jobs: - name: Install shellcheck run: | sudo apt update && sudo apt install shellcheck + + + - name: Install hadolint + run: | + curl -L "$(curl -s https://api.github.com/repos/hadolint/hadolint/releases/latest | grep -o -E -m 1 "https://.+?/hadolint-Linux-x86_64")" > hadolint \ + && chmod +x hadolint && sudo mv hadolint /usr/bin/ + # Need to success pre-commit fix push - uses: actions/checkout@v2 with: diff --git a/.gitignore b/.gitignore index 0bbeada90..b4a01b639 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,409 @@ tests/results/* + + +# Created by https://www.toptal.com/developers/gitignore/api/vim,intellij+all,emacs,eclipse,visualstudiocode,notepadpp,venv,python +# Edit at https://www.toptal.com/developers/gitignore?templates=vim,intellij+all,emacs,eclipse,visualstudiocode,notepadpp,venv,python + +### Eclipse ### +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Uncomment this line if you wish to ignore the project description file. +# Typically, this file would be tracked if it contains build/dependency configurations: +#.project + +### Eclipse Patch ### +# Spring Boot Tooling +.sts4-cache/ + +### Emacs ### +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data + + +### Intellij+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Intellij+all Patch ### +# Ignores the whole .idea folder and all .iml files +# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 + +.idea/ + +# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 + +*.iml +modules.xml +.idea/misc.xml +*.ipr + +# Sonarlint plugin +.idea/sonarlint + +### NotepadPP ### +# Notepad++ backups # + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +### venv ### +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +[Bb]in +[Ii]nclude +[Ll]ib +[Ll]ib64 +[Ll]ocal +[Ss]cripts +pyvenv.cfg +pip-selfcheck.json + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# Support for Project snippet scope +!.vscode/*.code-snippets + +# End of https://www.toptal.com/developers/gitignore/api/vim,intellij+all,emacs,eclipse,visualstudiocode,notepadpp,venv,python diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 000000000..2e71bea99 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,4 @@ +ignored: + - DL3013 + - DL3018 + - DL3059 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f876ccd35..b876c08fd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.1.0 hooks: # Git style - id: check-added-large-files @@ -34,3 +34,9 @@ repos: - id: shfmt args: ['-l', '-i', '2', '-ci', '-sr', '-w'] - id: shellcheck + + +- repo: https://github.com/hadolint/hadolint + rev: v2.8.0 + hooks: + - id: hadolint diff --git a/CHANGELOG.md b/CHANGELOG.md index 64a8c49e1..9cead2da6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -289,7 +289,7 @@ All notable changes to this project will be documented in this file. - fix: Change terraform_validate hook functionality for subdirectories with terraform files ([#100](https://github.com/antonbabenko/pre-commit-terraform/issues/100)) -### +### configuration for the appropriate working directory. diff --git a/Dockerfile b/Dockerfile index 9266eb346..ff881ba23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,32 +1,8 @@ -ARG TAG=3.9.7-alpine3.14 +ARG TAG=3.10.1-alpine3.15 FROM python:${TAG} as builder -WORKDIR /bin_dir - -RUN apk add --no-cache \ - # Builder deps - curl \ - unzip && \ - # Upgrade pip for be able get latest Checkov - python3 -m pip install --upgrade pip - ARG PRE_COMMIT_VERSION=${PRE_COMMIT_VERSION:-latest} ARG TERRAFORM_VERSION=${TERRAFORM_VERSION:-latest} - -# Install pre-commit -RUN [ ${PRE_COMMIT_VERSION} = "latest" ] && pip3 install --no-cache-dir pre-commit \ - || pip3 install --no-cache-dir pre-commit==${PRE_COMMIT_VERSION} - -# Install terraform because pre-commit needs it -RUN if [ "${TERRAFORM_VERSION}" = "latest" ]; then \ - TERRAFORM_VERSION="$(curl -s https://api.github.com/repos/hashicorp/terraform/releases/latest | grep tag_name | grep -o -E -m 1 "[0-9.]+")" \ - ; fi && \ - curl -L "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" > terraform.zip && \ - unzip terraform.zip terraform && rm terraform.zip - -# -# Install tools -# ARG CHECKOV_VERSION=${CHECKOV_VERSION:-false} ARG INFRACOST_VERSION=${INFRACOST_VERSION:-false} ARG TERRAFORM_DOCS_VERSION=${TERRAFORM_DOCS_VERSION:-false} @@ -34,143 +10,126 @@ ARG TERRAGRUNT_VERSION=${TERRAGRUNT_VERSION:-false} ARG TERRASCAN_VERSION=${TERRASCAN_VERSION:-false} ARG TFLINT_VERSION=${TFLINT_VERSION:-false} ARG TFSEC_VERSION=${TFSEC_VERSION:-false} +ARG INSTALL_ALL=${INSTALL_ALL:-false} + +SHELL ["/bin/ash", "-eo", "pipefail", "-c"] + +RUN apk add --no-cache gcc libc-dev linux-headers git curl jq libffi-dev +# setup runtime venv +ENV VIRTUAL_ENV=/opt/venv +RUN python3 -m venv --system-site-packages $VIRTUAL_ENV +ENV OLD_PATH=$PATH +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN python3 -m pip install --no-cache-dir --upgrade pip + +# pre-commit +COPY ./docker-scripts/install_pre-commit.sh /docker-scripts/install_pre-commit.sh +RUN /docker-scripts/install_pre-commit.sh # Tricky thing to install all tools by set only one arg. # In RUN command below used `. /.env` <- this is sourcing vars that # specified in step below -ARG INSTALL_ALL=${INSTALL_ALL:-false} -RUN if [ "$INSTALL_ALL" != "false" ]; then \ - echo "export CHECKOV_VERSION=latest" >> /.env && \ - echo "export INFRACOST_VERSION=latest" >> /.env && \ - echo "export TERRAFORM_DOCS_VERSION=latest" >> /.env && \ - echo "export TERRAGRUNT_VERSION=latest" >> /.env && \ - echo "export TERRASCAN_VERSION=latest" >> /.env && \ - echo "export TFLINT_VERSION=latest" >> /.env && \ - echo "export TFSEC_VERSION=latest" >> /.env \ - ; else \ - touch /.env \ - ; fi +COPY ./docker-scripts/populate-env-arch.sh /docker-scripts/populate-env-arch.sh +RUN /docker-scripts/populate-env-arch.sh + +# Terraform +COPY ./docker-scripts/install_terraform.sh /docker-scripts/install_terraform.sh +RUN /docker-scripts/install_terraform.sh + +COPY ./docker-scripts/populate-env-versions.sh /docker-scripts/populate-env-versions.sh +RUN /docker-scripts/populate-env-versions.sh +# PyPI # Checkov -RUN . /.env && \ - if [ "$CHECKOV_VERSION" != "false" ]; then \ - ( \ - apk add --no-cache gcc libffi-dev musl-dev; \ - [ "$CHECKOV_VERSION" = "latest" ] && pip3 install --no-cache-dir checkov \ - || pip3 install --no-cache-dir checkov==${CHECKOV_VERSION}; \ - apk del gcc libffi-dev musl-dev \ - ) \ - ; fi +COPY ./docker-scripts/install_checkov.sh /docker-scripts/install_checkov.sh +RUN /docker-scripts/install_checkov.sh + +# GitHub + +COPY ./docker-scripts/install-from-github.sh /docker-scripts/install-from-github.sh # infracost -RUN . /.env && \ - if [ "$INFRACOST_VERSION" != "false" ]; then \ - ( \ - INFRACOST_RELEASES="https://api.github.com/repos/infracost/infracost/releases" && \ - [ "$INFRACOST_VERSION" = "latest" ] && curl -L "$(curl -s ${INFRACOST_RELEASES}/latest | grep -o -E -m 1 "https://.+?-linux-amd64.tar.gz")" > infracost.tgz \ - || curl -L "$(curl -s ${INFRACOST_RELEASES} | grep -o -E "https://.+?v${INFRACOST_VERSION}/infracost-linux-amd64.tar.gz")" > infracost.tgz \ - ) && tar -xzf infracost.tgz && rm infracost.tgz && mv infracost-linux-amd64 infracost \ - ; fi +COPY ./docker-scripts/install_infracost.sh /docker-scripts/install_infracost.sh +RUN /docker-scripts/install_infracost.sh # Terraform docs -RUN . /.env && \ - if [ "$TERRAFORM_DOCS_VERSION" != "false" ]; then \ - ( \ - TERRAFORM_DOCS_RELEASES="https://api.github.com/repos/terraform-docs/terraform-docs/releases" && \ - [ "$TERRAFORM_DOCS_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRAFORM_DOCS_RELEASES}/latest | grep -o -E -m 1 "https://.+?-linux-amd64.tar.gz")" > terraform-docs.tgz \ - || curl -L "$(curl -s ${TERRAFORM_DOCS_RELEASES} | grep -o -E "https://.+?v${TERRAFORM_DOCS_VERSION}-linux-amd64.tar.gz")" > terraform-docs.tgz \ - ) && tar -xzf terraform-docs.tgz terraform-docs && rm terraform-docs.tgz && chmod +x terraform-docs \ - ; fi +COPY ./docker-scripts/install_terraform-docs.sh /docker-scripts/install_terraform-docs.sh +RUN /docker-scripts/install_terraform-docs.sh # Terragrunt -RUN . /.env \ - && if [ "$TERRAGRUNT_VERSION" != "false" ]; then \ - ( \ - TERRAGRUNT_RELEASES="https://api.github.com/repos/gruntwork-io/terragrunt/releases" && \ - [ "$TERRAGRUNT_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRAGRUNT_RELEASES}/latest | grep -o -E -m 1 "https://.+?/terragrunt_linux_amd64")" > terragrunt \ - || curl -L "$(curl -s ${TERRAGRUNT_RELEASES} | grep -o -E -m 1 "https://.+?v${TERRAGRUNT_VERSION}/terragrunt_linux_amd64")" > terragrunt \ - ) && chmod +x terragrunt \ - ; fi - +COPY ./docker-scripts/install_terragrunt.sh /docker-scripts/install_terragrunt.sh +RUN /docker-scripts/install_terragrunt.sh # Terrascan -RUN . /.env && \ - if [ "$TERRASCAN_VERSION" != "false" ]; then \ - ( \ - TERRASCAN_RELEASES="https://api.github.com/repos/accurics/terrascan/releases" && \ - [ "$TERRASCAN_VERSION" = "latest" ] && curl -L "$(curl -s ${TERRASCAN_RELEASES}/latest | grep -o -E -m 1 "https://.+?_Linux_x86_64.tar.gz")" > terrascan.tar.gz \ - || curl -L "$(curl -s ${TERRASCAN_RELEASES} | grep -o -E "https://.+?${TERRASCAN_VERSION}_Linux_x86_64.tar.gz")" > terrascan.tar.gz \ - ) && tar -xzf terrascan.tar.gz terrascan && rm terrascan.tar.gz && \ - ./terrascan init \ - ; fi +COPY ./docker-scripts/install_terrascan.sh /docker-scripts/install_terrascan.sh +RUN /docker-scripts/install_terrascan.sh # TFLint -RUN . /.env && \ - if [ "$TFLINT_VERSION" != "false" ]; then \ - ( \ - TFLINT_RELEASES="https://api.github.com/repos/terraform-linters/tflint/releases" && \ - [ "$TFLINT_VERSION" = "latest" ] && curl -L "$(curl -s ${TFLINT_RELEASES}/latest | grep -o -E -m 1 "https://.+?_linux_amd64.zip")" > tflint.zip \ - || curl -L "$(curl -s ${TFLINT_RELEASES} | grep -o -E "https://.+?/v${TFLINT_VERSION}/tflint_linux_amd64.zip")" > tflint.zip \ - ) && unzip tflint.zip && rm tflint.zip \ - ; fi +COPY ./docker-scripts/install_tflint.sh /docker-scripts/install_tflint.sh +RUN /docker-scripts/install_tflint.sh # TFSec -RUN . /.env && \ - if [ "$TFSEC_VERSION" != "false" ]; then \ - ( \ - TFSEC_RELEASES="https://api.github.com/repos/aquasecurity/tfsec/releases" && \ - [ "$TFSEC_VERSION" = "latest" ] && curl -L "$(curl -s ${TFSEC_RELEASES}/latest | grep -o -E -m 1 "https://.+?/tfsec-linux-amd64")" > tfsec \ - || curl -L "$(curl -s ${TFSEC_RELEASES} | grep -o -E -m 1 "https://.+?v${TFSEC_VERSION}/tfsec-linux-amd64")" > tfsec \ - ) && chmod +x tfsec \ - ; fi +COPY ./docker-scripts/install_tfsec.sh /docker-scripts/install_tfsec.sh +RUN /docker-scripts/install_tfsec.sh -# Checking binaries versions and write it to debug file -RUN . /.env && \ - F=tools_versions_info && \ - pre-commit --version >> $F && \ - ./terraform --version | head -n 1 >> $F && \ - (if [ "$CHECKOV_VERSION" != "false" ]; then echo "checkov $(checkov --version)" >> $F; else echo "checkov SKIPPED" >> $F ; fi) && \ - (if [ "$INFRACOST_VERSION" != "false" ]; then echo "$(./infracost --version)" >> $F; else echo "infracost SKIPPED" >> $F ; fi) && \ - (if [ "$TERRAFORM_DOCS_VERSION" != "false" ]; then ./terraform-docs --version >> $F; else echo "terraform-docs SKIPPED" >> $F ; fi) && \ - (if [ "$TERRAGRUNT_VERSION" != "false" ]; then ./terragrunt --version >> $F; else echo "terragrunt SKIPPED" >> $F ; fi) && \ - (if [ "$TERRASCAN_VERSION" != "false" ]; then echo "terrascan $(./terrascan version)" >> $F; else echo "terrascan SKIPPED" >> $F ; fi) && \ - (if [ "$TFLINT_VERSION" != "false" ]; then ./tflint --version >> $F; else echo "tflint SKIPPED" >> $F ; fi) && \ - (if [ "$TFSEC_VERSION" != "false" ]; then echo "tfsec $(./tfsec --version)" >> $F; else echo "tfsec SKIPPED" >> $F ; fi) && \ - echo -e "\n\n" && cat $F && echo -e "\n\n" +# setup build venv +ENV VIRTUAL_ENV=/opt/build-venv +ENV PATH="$OLD_PATH" +RUN python3 -m venv --system-site-packages $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +# install build tools +RUN python3 -m pip install --no-cache-dir --upgrade pip +RUN pip install --no-cache-dir --upgrade setuptools wheel build + +# build +COPY . /src/ +WORKDIR /src +RUN python3 -m build --wheel +# switch back to runtime venv +ENV VIRTUAL_ENV=/opt/venv +ENV PATH="$VIRTUAL_ENV/bin:$OLD_PATH" +# install package +RUN pip install --no-cache-dir --disable-pip-version-check ./dist/*.whl +WORKDIR / + +# Checking binaries versions and write it to debug file +COPY ./docker-scripts/write_tools-versions-info.sh /docker-scripts/write_tools-versions-info.sh +RUN /docker-scripts/write_tools-versions-info.sh +# runtime image FROM python:${TAG} +ARG INFRACOST_VERSION=${INFRACOST_VERSION:-false} +ARG TERRAFORM_DOCS_VERSION=${TERRAFORM_DOCS_VERSION:-false} +ARG INSTALL_ALL=${INSTALL_ALL:-false} + +ENV PYTHONUNBUFFERED=1 + RUN apk add --no-cache \ # pre-commit deps git \ # All hooks deps - bash - -# Copy tools -COPY --from=builder \ - # Needed for all hooks - /usr/local/bin/pre-commit \ - # Hooks and terraform binaries - /bin_dir/ \ - /usr/local/bin/checkov* \ - /usr/bin/ -# Copy pre-commit packages -COPY --from=builder /usr/local/lib/python3.9/site-packages/ /usr/local/lib/python3.9/site-packages/ -# Copy terrascan policies -COPY --from=builder /root/ /root/ - -# Install hooks extra deps -RUN if [ "$(grep -o '^terraform-docs SKIPPED$' /usr/bin/tools_versions_info)" = "" ]; then \ + bash && \ + if [ "${TERRAFORM_DOCS_VERSION}" != "false" ] || [ "${INSTALL_ALL}" != "false" ]; then \ apk add --no-cache perl \ ; fi && \ - if [ "$(grep -o '^infracost SKIPPED$' /usr/bin/tools_versions_info)" = "" ]; then \ + if [ "${INFRACOST_VERSION}" != "false" ] || [ "${INSTALL_ALL}" != "false" ]; then \ apk add --no-cache jq \ ; fi +# Copy terrascan policies +COPY --from=builder /root/.terrascan/pkg/policies/opa/rego/ /root/.terrascan/pkg/policies/opa/rego/ + +# copy venv +ENV VIRTUAL_ENV=/opt/venv +COPY --from=builder /opt/venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + ENV PRE_COMMIT_COLOR=${PRE_COMMIT_COLOR:-always} ENV INFRACOST_API_KEY=${INFRACOST_API_KEY:-} diff --git a/docker-scripts/install-from-github.sh b/docker-scripts/install-from-github.sh new file mode 100755 index 000000000..f1c9c732e --- /dev/null +++ b/docker-scripts/install-from-github.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env ash +# shellcheck shell=dash + +set -euo pipefail + +PCT_ASSET_INCLUDE_VERSION="${PCT_ASSET_INCLUDE_VERSION:-true}" +PCT_SUFFIX="${PCT_SUFFIX:-}" +PCT_ASSET_BIN_NAME="${PCT_ASSET_BIN_NAME:-${PCT_BIN_NAME}}" +PCT_VERSION_PARAM="${PCT_VERSION_PARAM:-"--version"}" + +if [ "${PCT_VERSION}" = "latest" ]; then + PCT_TAG="$(curl -sSfL "https://api.github.com/repos/${PCT_GITHUB_USER}/${PCT_GITHUB_PROJECT}/releases/latest" | jq -r '.tag_name')" + PCT_VERSION=$(echo "${PCT_TAG}" | grep -o -E -m 1 "[0-9.]+") +else + PCT_TAG="${PCT_TAG_PREFIX:-v}${PCT_VERSION}" +fi + +if [ "${PCT_ASSET_INCLUDE_VERSION}" = "false" ]; then + PCT_VERSION="" +fi + +PCT_GITHUB_URL="https://github.com/${PCT_GITHUB_USER}/${PCT_GITHUB_PROJECT}/releases/download/${PCT_TAG}/${PCT_PREFIX}${PCT_VERSION}${PCT_INFIX}${PCT_ARCH}${PCT_SUFFIX}" + +echo Downloading... +curl -sSfL "${PCT_GITHUB_URL}" > "${PCT_BIN_NAME}${PCT_SUFFIX}" +echo Downloaded + +case "$PCT_SUFFIX" in + .zip) + unzip -q "${PCT_BIN_NAME}${PCT_SUFFIX}" "${PCT_ASSET_BIN_NAME}" + rm "${PCT_BIN_NAME}${PCT_SUFFIX}" + ;; + .tgz | .tar.gz) + tar -xzf "${PCT_BIN_NAME}${PCT_SUFFIX}" "${PCT_ASSET_BIN_NAME}" + rm "${PCT_BIN_NAME}${PCT_SUFFIX}" + ;; +esac + +chmod a+x "${PCT_ASSET_BIN_NAME}" +mv "${PCT_ASSET_BIN_NAME}" "${VIRTUAL_ENV}/bin/${PCT_BIN_NAME}" + +"${PCT_BIN_NAME}" "${PCT_VERSION_PARAM}" diff --git a/docker-scripts/install_checkov.sh b/docker-scripts/install_checkov.sh new file mode 100755 index 000000000..fd4397a97 --- /dev/null +++ b/docker-scripts/install_checkov.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env ash +# shellcheck shell=dash + +set -euo pipefail + +. /.env +if [ "$CHECKOV_VERSION" != "false" ]; then + if [ "$CHECKOV_VERSION" = "latest" ]; then + pip3 install --no-cache-dir checkov + else + pip3 install --no-cache-dir "checkov==${CHECKOV_VERSION}" + fi + # reinstall latest pip + python3 -m pip install --no-cache-dir --upgrade pip +fi diff --git a/docker-scripts/install_infracost.sh b/docker-scripts/install_infracost.sh new file mode 100755 index 000000000..96c82b4de --- /dev/null +++ b/docker-scripts/install_infracost.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env ash +# shellcheck shell=dash + +set -euo pipefail + +. /.env +if [ "$INFRACOST_VERSION" != "false" ]; then + export PCT_VERSION=$INFRACOST_VERSION + export PCT_GITHUB_USER=infracost + export PCT_GITHUB_PROJECT=infracost + export PCT_BIN_NAME=infracost + export PCT_PREFIX="${PCT_BIN_NAME}-" + export PCT_ASSET_INCLUDE_VERSION=false + export PCT_INFIX="linux-" + export PCT_ARCH=${ARCH} + export PCT_SUFFIX=".tar.gz" + export PCT_ASSET_BIN_NAME="infracost-linux-amd64" + /docker-scripts/install-from-github.sh +fi diff --git a/docker-scripts/install_pre-commit.sh b/docker-scripts/install_pre-commit.sh new file mode 100755 index 000000000..ed068b0b4 --- /dev/null +++ b/docker-scripts/install_pre-commit.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env ash +# shellcheck shell=dash + +set -euo pipefail + +if [ "${PRE_COMMIT_VERSION}" = "latest" ]; then + pip3 install --no-cache-dir pre-commit +else + pip3 install --no-cache-dir "pre-commit==${PRE_COMMIT_VERSION}" +fi + +# reinstall latest pip +python3 -m pip install --no-cache-dir --upgrade pip diff --git a/docker-scripts/install_terraform-docs.sh b/docker-scripts/install_terraform-docs.sh new file mode 100755 index 000000000..2f5a7432c --- /dev/null +++ b/docker-scripts/install_terraform-docs.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env ash +# shellcheck shell=dash + +set -euo pipefail + +. /.env +if [ "$TERRAFORM_DOCS_VERSION" != "false" ]; then + export PCT_VERSION=$TERRAFORM_DOCS_VERSION + export PCT_GITHUB_USER=terraform-docs + export PCT_GITHUB_PROJECT=terraform-docs + export PCT_BIN_NAME=terraform-docs + export PCT_PREFIX="${PCT_BIN_NAME}-v" + export PCT_INFIX="-linux-" + export PCT_ARCH=${ARCH} + export PCT_SUFFIX=.tar.gz + /docker-scripts/install-from-github.sh +fi diff --git a/docker-scripts/install_terraform.sh b/docker-scripts/install_terraform.sh new file mode 100755 index 000000000..7587d2ac7 --- /dev/null +++ b/docker-scripts/install_terraform.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env ash +# shellcheck shell=dash + +set -euo pipefail + +. /.env +if [ "${TERRAFORM_VERSION}" = "latest" ]; then + TERRAFORM_VERSION="$(curl -sSfL https://api.github.com/repos/hashicorp/terraform/releases/latest | jq -r '.tag_name' | grep -o -E -m 1 "[0-9.]+")" +fi + +echo Downloading... +curl -sSfL "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_${ARCH}.zip" -o terraform.zip +echo Downloaded + +unzip -q terraform.zip terraform +chmod a+x terraform +mv terraform "${VIRTUAL_ENV}/bin/" +rm terraform.zip diff --git a/docker-scripts/install_terragrunt.sh b/docker-scripts/install_terragrunt.sh new file mode 100755 index 000000000..29d2db4b1 --- /dev/null +++ b/docker-scripts/install_terragrunt.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env ash +# shellcheck shell=dash + +set -euo pipefail + +. /.env +if [ "$TERRAGRUNT_VERSION" != "false" ]; then + export PCT_VERSION=$TERRAGRUNT_VERSION + export PCT_GITHUB_USER=gruntwork-io + export PCT_GITHUB_PROJECT=terragrunt + export PCT_BIN_NAME=terragrunt + export PCT_PREFIX="${PCT_BIN_NAME}" + export PCT_ASSET_INCLUDE_VERSION=false + export PCT_INFIX="_linux_" + export PCT_ARCH=${ARCH} + /docker-scripts/install-from-github.sh +fi diff --git a/docker-scripts/install_terrascan.sh b/docker-scripts/install_terrascan.sh new file mode 100755 index 000000000..7298bb714 --- /dev/null +++ b/docker-scripts/install_terrascan.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env ash +# shellcheck shell=dash + +set -euo pipefail + +mkdir -p /root/.terrascan/pkg/policies/opa/rego/ + +. /.env +if [ "$TERRASCAN_VERSION" != "false" ]; then + export PCT_VERSION=$TERRASCAN_VERSION + export PCT_GITHUB_USER=accurics + export PCT_GITHUB_PROJECT=terrascan + export PCT_BIN_NAME=terrascan + export PCT_PREFIX="${PCT_BIN_NAME}_" + export PCT_INFIX="_Linux_" + export PCT_ARCH=${ARCHX} + export PCT_SUFFIX=.tar.gz + export PCT_VERSION_PARAM="version" + /docker-scripts/install-from-github.sh + echo terrascan init + terrascan init +fi diff --git a/docker-scripts/install_tflint.sh b/docker-scripts/install_tflint.sh new file mode 100755 index 000000000..5609b912b --- /dev/null +++ b/docker-scripts/install_tflint.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env ash +# shellcheck shell=dash + +set -euo pipefail + +. /.env +if [ "$TFLINT_VERSION" != "false" ]; then + export PCT_VERSION=$TFLINT_VERSION + export PCT_GITHUB_USER=terraform-linters + export PCT_GITHUB_PROJECT=tflint + export PCT_BIN_NAME=tflint + export PCT_PREFIX="${PCT_BIN_NAME}" + export PCT_ASSET_INCLUDE_VERSION=false + export PCT_INFIX="_linux_" + export PCT_ARCH=${ARCH} + export PCT_SUFFIX=.zip + /docker-scripts/install-from-github.sh +fi diff --git a/docker-scripts/install_tfsec.sh b/docker-scripts/install_tfsec.sh new file mode 100755 index 000000000..9214f085b --- /dev/null +++ b/docker-scripts/install_tfsec.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env ash +# shellcheck shell=dash + +set -euo pipefail + +. /.env +if [ "$TFSEC_VERSION" != "false" ]; then + export PCT_VERSION=$TFSEC_VERSION + export PCT_GITHUB_USER=aquasecurity + export PCT_GITHUB_PROJECT=tfsec + export PCT_BIN_NAME=tfsec + export PCT_PREFIX="${PCT_BIN_NAME}" + export PCT_ASSET_INCLUDE_VERSION=false + export PCT_INFIX="-linux-" + export PCT_ARCH=${ARCH} + /docker-scripts/install-from-github.sh +fi diff --git a/docker-scripts/populate-env-arch.sh b/docker-scripts/populate-env-arch.sh new file mode 100755 index 000000000..a77c6db5c --- /dev/null +++ b/docker-scripts/populate-env-arch.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env ash +# shellcheck shell=dash + +set -euo pipefail + +case $(uname -m) in + armv7l) + { + echo "export ARCH=arm64" + echo "export ARCHX=arm64" + } >> /.env + ;; + x86_64) + { + echo "export ARCH=amd64" + echo "export ARCHX=x86_64" + } >> /.env + ;; + *) + exit 1 + ;; +esac diff --git a/docker-scripts/populate-env-versions.sh b/docker-scripts/populate-env-versions.sh new file mode 100755 index 000000000..a1339d1f4 --- /dev/null +++ b/docker-scripts/populate-env-versions.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env ash +# shellcheck shell=dash + +set -euo pipefail + +override_or_latest() { + if [ "$1" = "false" ]; then + echo -n "latest" + else + echo -n "$1" + fi +} + +if [ "$INSTALL_ALL" != "false" ]; then + { + echo "export CHECKOV_VERSION=$(override_or_latest "${CHECKOV_VERSION}")" + echo "export INFRACOST_VERSION=$(override_or_latest "${INFRACOST_VERSION}")" + echo "export TERRAFORM_DOCS_VERSION=$(override_or_latest "${TERRAFORM_DOCS_VERSION}")" + echo "export TERRAGRUNT_VERSION=$(override_or_latest "${TERRAGRUNT_VERSION}")" + echo "export TERRASCAN_VERSION=$(override_or_latest "${TERRASCAN_VERSION}")" + echo "export TFLINT_VERSION=$(override_or_latest "${TFLINT_VERSION}")" + echo "export TFSEC_VERSION=$(override_or_latest "${TFSEC_VERSION}")" + } >> /.env +else + { + echo "export CHECKOV_VERSION=${CHECKOV_VERSION}" + echo "export INFRACOST_VERSION=${INFRACOST_VERSION}" + echo "export TERRAFORM_DOCS_VERSION=${TERRAFORM_DOCS_VERSION}" + echo "export TERRAGRUNT_VERSION=${TERRAGRUNT_VERSION}" + echo "export TERRASCAN_VERSION=${TERRASCAN_VERSION}" + echo "export TFLINT_VERSION=${TFLINT_VERSION}" + echo "export TFSEC_VERSION=${TFSEC_VERSION}" + } >> /.env +fi diff --git a/docker-scripts/write_tools-versions-info.sh b/docker-scripts/write_tools-versions-info.sh new file mode 100755 index 000000000..d4eb11c52 --- /dev/null +++ b/docker-scripts/write_tools-versions-info.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env ash +# shellcheck shell=dash + +set -euo pipefail + +. /.env +F="${VIRTUAL_ENV}/tools_versions_info" +{ + pre-commit --version + terraform --version | head -n 1 + (if [ "$CHECKOV_VERSION" != "false" ]; then echo "checkov $(checkov --version)"; else echo "checkov SKIPPED"; fi) + (if [ "$INFRACOST_VERSION" != "false" ]; then infracost --version; else echo "infracost SKIPPED"; fi) + (if [ "$TERRAFORM_DOCS_VERSION" != "false" ]; then terraform-docs --version; else echo "terraform-docs SKIPPED"; fi) + (if [ "$TERRAGRUNT_VERSION" != "false" ]; then terragrunt --version; else echo "terragrunt SKIPPED"; fi) + (if [ "$TERRASCAN_VERSION" != "false" ]; then echo "terrascan $(terrascan version)"; else echo "terrascan SKIPPED"; fi) + (if [ "$TFLINT_VERSION" != "false" ]; then tflint --version; else echo "tflint SKIPPED"; fi) + (if [ "$TFSEC_VERSION" != "false" ]; then echo "tfsec $(tfsec --version)"; else echo "tfsec SKIPPED"; fi) +} >> "$F" +printf '\n\n' && cat "$F" && printf '\n\n' diff --git a/hooks/infracost_breakdown.sh b/hooks/infracost_breakdown.sh index 911bcacaf..f4c4f1c5d 100755 --- a/hooks/infracost_breakdown.sh +++ b/hooks/infracost_breakdown.sh @@ -45,8 +45,7 @@ function infracost_breakdown_ { # $hook_config receives string like '1 > 2; 3 == 4;' etc. # It gets split by `;` into array, which we're parsing here ('1 > 2' ' 3 == 4') # Next line removes leading spaces, just for fancy output reason. - # shellcheck disable=SC2001 # Rule exception - check=$(echo "$check" | sed 's/^[[:space:]]*//') + check=${check//^[[:space:]]*//} # Drop quotes in hook args section. From: # -h ".totalHourlyCost > 0.1" @@ -61,8 +60,8 @@ function infracost_breakdown_ { }; then check="${check:1:-1}" fi - # shellcheck disable=SC2207 # Can't find working `read` command - operations=($(echo "$check" | grep -oE '[!<>=]{1,2}')) + + mapfile -t operations < <(echo "$check" | grep -oE '[!<>=]{1,2}') # Get the very last operator, that is used in comparison inside `jq` query. # From the example below we need to pick the `>` which is in between `add` and `1000`, # but not the `!=`, which goes earlier in the `jq` expression diff --git a/setup.py b/setup.py index 2d88425b9..1022ccce3 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ ], packages=find_packages(exclude=('tests*', 'testing*')), - install_requires=[ + setup_requires=[ 'setuptools-git-version', ], entry_points={ diff --git a/tests/Dockerfile b/tests/Dockerfile index cc0f7d2e1..b36b288ff 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,7 +1,9 @@ +# hadolint ignore=DL3006 FROM pre-commit -RUN apt update && \ - apt install -y \ +# hadolint ignore=DL3008 +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ datamash \ time && \ # Cleanup