From f91491c2081828986e227aadfe224f0dfda15ede Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 18 Jan 2024 00:22:57 +0100 Subject: [PATCH 01/13] Draft: Configure image to run as non-root by default --- Dockerfile | 112 +++++++++++++++++++----------------------------- README.md | 9 ++-- formulae/buf.py | 9 ++++ 3 files changed, 57 insertions(+), 73 deletions(-) create mode 100644 formulae/buf.py diff --git a/Dockerfile b/Dockerfile index 81d66f2..fbbece2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,24 +2,24 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} ARG BASE_IMAGE - -ENV DEBIAN_FRONTEND noninteractive -RUN : \ - && apt-get update \ - && apt-get install -y curl git wget libssl-dev libffi-dev llvm clang gcc g++ pkg-config build-essential jq sudo \ - && rm -rf /var/cache/apt/archives /var/lib/apt/lists/* - -# Install Python versions with deadsnakes. SHELL [ "/bin/bash", "-c" ] + RUN : \ && set -x \ + && export DEBIAN_FRONTEND=noninteractive \ && apt-get update \ - && apt-get install -y software-properties-common --no-install-recommends \ + && apt-get install -y curl \ + && curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \ + && apt update \ + && apt-get install -y git wget libssl-dev libffi-dev llvm clang gcc g++ pkg-config build-essential jq sudo software-properties-common graphviz unzip lcov git-lfs docker.io nodejs xxd cmake --no-install-recommends \ && add-apt-repository ppa:deadsnakes/ppa \ && apt update \ && apt-get install -y python{3.8,3.9,3.10,3.11,3.12}{,-venv,-dev} --no-install-recommends \ && rm -rf /var/cache/apt/archives /var/lib/apt/lists/* +# Install docker-buildx +COPY --from=docker/buildx-bin:latest /buildx /usr/libexec/docker/cli-plugins/docker-buildx + RUN : \ # Install Pip for all other Python versions. && set -x \ @@ -32,14 +32,32 @@ RUN : \ && ln -svf $(which python3.10) /usr/bin/python \ && ln -svf $(which python3.10) /usr/bin/python3 -ENV PATH="$PATH:/root/.cargo/bin:/root/.local/bin" - -COPY formulae /tmp/formulae -COPY src /tmp/src +# Nix RUN : \ - # - # install from custom formulae - # + && sh <(curl -L https://nixos.org/nix/install) --daemon \ + && echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf \ + && echo "max-jobs = auto" >> /etc/nix/nix.conf + +# Setup the main user which can run apt-get. +RUN useradd -m -s /bin/bash -G sudo,root -u 1000 main +RUN echo "main ALL=NOPASSWD: $(which apt-get)" >> /etc/sudoers +RUN chown main:main -R /usr/local /opt +USER main + +# Point CARGO_HOME and PIPX_HOME outside of the home directory as that may get overridden with a mount. +ENV CARGO_HOME=/opt/cargo +ENV PIPX_HOME=/opt/pipx +ENV PIPX_BIN_DIR=/usr/local/bin +ENV PIPX_MAN_DIR=/usr/local/share/man +ENV PATH="$PATH:/$CARGO_HOME/bin" + +# At runtime many tools install to $HOME/.local/bin, but we don't as home may get overriden. +ENV PATH="$PATH:/$HOME/.local/bin" + +RUN --mount=bind,src=formulae,target=/tmp/formulae \ + --mount=bind,src=formulae,target=/tmp/src : \ + && set -x \ + && python /tmp/src/main.py /tmp/formulae/buf.py \ && python /tmp/src/main.py /tmp/formulae/buildkit.py \ && python /tmp/src/main.py /tmp/formulae/grcov.py \ && python /tmp/src/main.py /tmp/formulae/kubectl.py \ @@ -48,35 +66,10 @@ RUN : \ && python /tmp/src/main.py /tmp/formulae/sccache.py \ && python /tmp/src/main.py /tmp/formulae/terraform.py \ && python /tmp/src/main.py /tmp/formulae/yq.py \ - # - # more APT packages - # - && curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \ - && apt-get update \ - && apt-get install -y docker.io nodejs graphviz unzip lcov git-lfs \ - # - # Rust - # - && apt-get install -y xxd cmake \ - && ( curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ) \ - # - # helm - # && ( curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash ) \ - # - # [cleanup] - # - && rm -r /tmp/src /tmp/formulae \ - && rm -rf ~/.cache /var/cache/apt/archives /var/lib/apt/lists/* - -# -# docker-buildx -# -COPY --from=docker/buildx-bin:latest /buildx /usr/libexec/docker/cli-plugins/docker-buildx + && ( curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ) -# -# Rust tools -# +# Rust tools, build with sccache and GHA cache to speed up builds. ARG ACTIONS_CACHE_URL RUN --mount=type=secret,id=ACTIONS_RUNTIME_TOKEN : \ && rustup toolchain install 1.75.0 \ @@ -95,35 +88,16 @@ RUN --mount=type=secret,id=ACTIONS_RUNTIME_TOKEN : \ && cargo install cargo-llvm-cov --version 0.5.39 \ && cargo install cargo-hack --version 0.6.15 \ && cargo install buffrs --version 0.7.5 \ - && sccache --stop-server + && cargo install cargo-cache --version 0.8.3 \ + && sccache --stop-server \ + && cargo cache --autoclean -# -# Buf (for Buffrs) -# -RUN : \ - && BIN="/usr/bin" \ - && VERSION="1.17.0" \ - && curl -sSL \ - "https://github.com/bufbuild/buf/releases/download/v${VERSION}/buf-$(uname -s)-$(uname -m)" \ - -o "${BIN}/buf" && \ - chmod +x "${BIN}/buf" - -# # Python tools -# RUN : \ - && python -m pip install pipx==1.3.3 -v \ + && python -m pip install pipx==1.4.3 -v \ && pipx install poetry==1.7.1 \ - && pipx install pdm==2.11.1 \ - && pipx install slap-cli==1.11.1 \ - && pipx install kraken-wrapper==0.32.4 \ + && pipx install pdm==2.12.1 \ + && pipx install slap-cli==1.11.2 \ + && pipx install kraken-wrapper==0.33.1 \ && pipx install ansible-base==2.10.17 && pipx inject ansible-base ansible==9.1.0 \ && rm -rf ~/.cache/pip - -# -# Nix -# -RUN : \ - && sh <(curl -L https://nixos.org/nix/install) --daemon \ - && echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf \ - && echo "max-jobs = auto" >> /etc/nix/nix.conf diff --git a/README.md b/README.md index 6f3a3fe..472ead0 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ the base image in that minor version range besides a higher minor having already | buffrs | cargo | 0.7.5 | | build-essential | apt-get | latest | | BuildKit | GitHub Releases | 0.12.4 | +| cargo-cache | cargo | 0.8.3 | | cargo-deny | cargo | 0.14.3 | | cargo-hack | cargo | 0.6.15 | | cargo-llvm-cov | cargo | 0.5.39 | @@ -53,7 +54,7 @@ the base image in that minor version range besides a higher minor having already | grcov | [GitHub releases](https://github.com/mozilla/grcov/releases) ([formula](formulae/grcov.py)) | 0.8.19 | | jq | apt-get | latest | | Helm | get-helm-3 | latest | -| kraken-wrapper | Pipx (Python 3.10) | 0.32.4 | +| kraken-wrapper | Pipx (Python 3.10) | 0.33.1 | | Kubectl | apt-get (`apt.kubernetes.io`) | 1.28.4 | | lcov | apt-get | latest | | libffi | apt-get | latest | @@ -62,8 +63,8 @@ the base image in that minor version range besides a higher minor having already | manifest-tool | [GitHub releases](https://github.com/estesp/manifest-tool/releases) ([formula](formulae/manifest-tool.py)) | 2.1.5 | | Nix | `https://nixos.org/nix/install` | latest | | NodeJS | apt-get (via [nodesource install](https://github.com/nodesource/distributions#debinstall)) | 18 | -| PDM | Pipx (Python 3.10) | 2.11.1 | -| Pipx | Pip (Python 3.10) | 1.3.3 | +| PDM | Pipx (Python 3.10) | 2.12.1 | +| Pipx | Pip (Python 3.10) | 1.4.3 | | pkg-config | apt-get | latest | | Poetry | Pipx (Python 3.10) | 1.7.1 | | protobuf-compiler | [GitHub releases](https://github.com/protocolbuffers/protobuf/releases) ([formula](formulae/protobuf-compiler.py)) | 3.20.1 | @@ -72,7 +73,7 @@ the base image in that minor version range besides a higher minor having already | Rustup | rustup.rs | latest | | rustfmt | rustup | nightly (additionally) | | sccache | [GitHub releases](https://github.com/mozilla/sccache/releases) ([formula](formulae/sccache.py)) | 0.7.4 | -| Slap ([link](https://github.com/python-slap/slap-cli)) | Pipx (Python 3.10) | 1.11.1 | +| Slap ([link](https://github.com/python-slap/slap-cli)) | Pipx (Python 3.10) | 1.11.2 | | sqlx-cli | cargo | 0.7.3 | | Terraform | Hashicorp releases | 1.6.6 | | wget | apt-get | latest | diff --git a/formulae/buf.py b/formulae/buf.py new file mode 100644 index 0000000..c804069 --- /dev/null +++ b/formulae/buf.py @@ -0,0 +1,9 @@ +from formula import DownloadFileFormula + + +class BufFormula(DownloadFileFormula): + + version = "1.17.0" + download_url = "https://github.com/bufbuild/buf/releases/download/v${version}/buf-Linux-{archv1}" + output_file = "/usr/local/bin/buf" + chmod = 0o755 From 2adb599e65bbddf13fa867d0e7ef60542bc31172 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 18 Jan 2024 00:29:52 +0100 Subject: [PATCH 02/13] fix buf download url template --- formulae/buf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/formulae/buf.py b/formulae/buf.py index c804069..4ce95c8 100644 --- a/formulae/buf.py +++ b/formulae/buf.py @@ -4,6 +4,6 @@ class BufFormula(DownloadFileFormula): version = "1.17.0" - download_url = "https://github.com/bufbuild/buf/releases/download/v${version}/buf-Linux-{archv1}" + download_url = "https://github.com/bufbuild/buf/releases/download/v${version}/buf-Linux-${archv1}" output_file = "/usr/local/bin/buf" chmod = 0o755 From 24137ffed3a81a013898ebacb22765a86b1bc202 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 18 Jan 2024 01:04:06 +0100 Subject: [PATCH 03/13] respect install_to in BufFormula --- formulae/buf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/formulae/buf.py b/formulae/buf.py index 4ce95c8..51d3d54 100644 --- a/formulae/buf.py +++ b/formulae/buf.py @@ -5,5 +5,5 @@ class BufFormula(DownloadFileFormula): version = "1.17.0" download_url = "https://github.com/bufbuild/buf/releases/download/v${version}/buf-Linux-${archv1}" - output_file = "/usr/local/bin/buf" + output_file = "{install_to}/buf" chmod = 0o755 From 59a07531fbfe5cd858bb9a3b9976dedcef91f6c7 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 18 Jan 2024 01:04:23 +0100 Subject: [PATCH 04/13] set default install_to in BufFormula --- formulae/buf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/formulae/buf.py b/formulae/buf.py index 51d3d54..f5958c8 100644 --- a/formulae/buf.py +++ b/formulae/buf.py @@ -5,5 +5,6 @@ class BufFormula(DownloadFileFormula): version = "1.17.0" download_url = "https://github.com/bufbuild/buf/releases/download/v${version}/buf-Linux-${archv1}" + install_to = "/usr/local/bin" output_file = "{install_to}/buf" chmod = 0o755 From 122fc85e26e02dfc03047effa5cbeb0e29daf021 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 18 Jan 2024 01:31:20 +0100 Subject: [PATCH 05/13] fix bind mount --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fbbece2..9be70ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,8 +54,8 @@ ENV PATH="$PATH:/$CARGO_HOME/bin" # At runtime many tools install to $HOME/.local/bin, but we don't as home may get overriden. ENV PATH="$PATH:/$HOME/.local/bin" -RUN --mount=bind,src=formulae,target=/tmp/formulae \ - --mount=bind,src=formulae,target=/tmp/src : \ +RUN --mount=type=bind,src=formulae,target=/tmp/formulae \ + --mount=type=bind,src=formulae,target=/tmp/src : \ && set -x \ && python /tmp/src/main.py /tmp/formulae/buf.py \ && python /tmp/src/main.py /tmp/formulae/buildkit.py \ From 6725812b0dfe670c6b2cd794246e34fed9d0560f Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 18 Jan 2024 02:41:04 +0100 Subject: [PATCH 06/13] fix bind mount --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9be70ed..c16407d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,7 +55,7 @@ ENV PATH="$PATH:/$CARGO_HOME/bin" ENV PATH="$PATH:/$HOME/.local/bin" RUN --mount=type=bind,src=formulae,target=/tmp/formulae \ - --mount=type=bind,src=formulae,target=/tmp/src : \ + --mount=type=bind,src=src,target=/tmp/src : \ && set -x \ && python /tmp/src/main.py /tmp/formulae/buf.py \ && python /tmp/src/main.py /tmp/formulae/buildkit.py \ From b8a1ccb7958ed5690cbc2cea6f167165a5e1634a Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Thu, 18 Jan 2024 21:38:35 +0100 Subject: [PATCH 07/13] fix output_file template --- formulae/buf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/formulae/buf.py b/formulae/buf.py index f5958c8..4e05ea4 100644 --- a/formulae/buf.py +++ b/formulae/buf.py @@ -6,5 +6,5 @@ class BufFormula(DownloadFileFormula): version = "1.17.0" download_url = "https://github.com/bufbuild/buf/releases/download/v${version}/buf-Linux-${archv1}" install_to = "/usr/local/bin" - output_file = "{install_to}/buf" + output_file = "${install_to}/buf" chmod = 0o755 From 63f7724af36000e0c370aac56a110cf74145093f Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 19 Jan 2024 18:20:50 +0100 Subject: [PATCH 08/13] add --no-root to helm install script; --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c16407d..24ebcb2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,7 +66,7 @@ RUN --mount=type=bind,src=formulae,target=/tmp/formulae \ && python /tmp/src/main.py /tmp/formulae/sccache.py \ && python /tmp/src/main.py /tmp/formulae/terraform.py \ && python /tmp/src/main.py /tmp/formulae/yq.py \ - && ( curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash ) \ + && ( curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 --no-root | bash ) \ && ( curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ) # Rust tools, build with sccache and GHA cache to speed up builds. From a0c41ca5e4b279d939c4befaa54a97a573734465 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 19 Jan 2024 23:51:48 +0100 Subject: [PATCH 09/13] set build secret uid --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 24ebcb2..054c759 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,7 +71,7 @@ RUN --mount=type=bind,src=formulae,target=/tmp/formulae \ # Rust tools, build with sccache and GHA cache to speed up builds. ARG ACTIONS_CACHE_URL -RUN --mount=type=secret,id=ACTIONS_RUNTIME_TOKEN : \ +RUN --mount=type=secret,id=ACTIONS_RUNTIME_TOKEN,uid=1000 : \ && rustup toolchain install 1.75.0 \ && rustup toolchain install nightly --component rustfmt \ && rustup default 1.75.0 \ From 72f9b75948b5a514d6a32c92183c813af5cd9e35 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Sat, 20 Jan 2024 00:32:02 +0100 Subject: [PATCH 10/13] remove main user from sudo,root groups --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 054c759..a51bc95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,7 @@ RUN : \ && echo "max-jobs = auto" >> /etc/nix/nix.conf # Setup the main user which can run apt-get. -RUN useradd -m -s /bin/bash -G sudo,root -u 1000 main +RUN useradd -m -s /bin/bash -u 1000 main RUN echo "main ALL=NOPASSWD: $(which apt-get)" >> /etc/sudoers RUN chown main:main -R /usr/local /opt USER main From d98862ef7f4b21a6ad17b649ab14d072bacd864d Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Sat, 20 Jan 2024 00:40:11 +0100 Subject: [PATCH 11/13] change workdir to /home/main --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index a51bc95..4b44aab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,7 @@ RUN useradd -m -s /bin/bash -u 1000 main RUN echo "main ALL=NOPASSWD: $(which apt-get)" >> /etc/sudoers RUN chown main:main -R /usr/local /opt USER main +WORKDIR /home/main # Point CARGO_HOME and PIPX_HOME outside of the home directory as that may get overridden with a mount. ENV CARGO_HOME=/opt/cargo From a77629e004877023c936a889d2a4b117b376cfab Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Sat, 20 Jan 2024 01:16:45 +0100 Subject: [PATCH 12/13] set RUSTUP_HOME --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 4b44aab..ff7e100 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,6 +46,7 @@ USER main WORKDIR /home/main # Point CARGO_HOME and PIPX_HOME outside of the home directory as that may get overridden with a mount. +ENV RUSTUP_HOME=/opt/rustup ENV CARGO_HOME=/opt/cargo ENV PIPX_HOME=/opt/pipx ENV PIPX_BIN_DIR=/usr/local/bin From 4f796b0f998aa67602e3b6f9179b8435df87d130 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 23 Jan 2024 10:47:49 +0100 Subject: [PATCH 13/13] use uid 10000 (https://github.com/dnaprawa/dockerfile-best-practices#do-not-use-a-uid-below-10000) --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ff7e100..e603ebd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,7 @@ RUN : \ && echo "max-jobs = auto" >> /etc/nix/nix.conf # Setup the main user which can run apt-get. -RUN useradd -m -s /bin/bash -u 1000 main +RUN useradd -m -s /bin/bash -u 10000 main RUN echo "main ALL=NOPASSWD: $(which apt-get)" >> /etc/sudoers RUN chown main:main -R /usr/local /opt USER main @@ -73,7 +73,7 @@ RUN --mount=type=bind,src=formulae,target=/tmp/formulae \ # Rust tools, build with sccache and GHA cache to speed up builds. ARG ACTIONS_CACHE_URL -RUN --mount=type=secret,id=ACTIONS_RUNTIME_TOKEN,uid=1000 : \ +RUN --mount=type=secret,id=ACTIONS_RUNTIME_TOKEN,uid=10000 : \ && rustup toolchain install 1.75.0 \ && rustup toolchain install nightly --component rustfmt \ && rustup default 1.75.0 \