From dd507443e925e16ed1f7a2879be92cb6e08f4a6d Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Mon, 2 Dec 2024 16:41:58 +1100
Subject: [PATCH 1/3] Add Docker image release

Also run the release pipeline in a separate BK pipeline
---
 .buildkite/Dockerfile.public             |  6 ++
 .buildkite/pipeline.release.yml          | 14 +++-
 .buildkite/pipeline.yml                  | 19 ++++-
 .buildkite/steps/release-docker.sh       | 89 ++++++++++++++++++++++++
 .buildkite/steps/release-github.sh       | 30 +++++---
 .buildkite/steps/release-s3-version.sh   |  5 ++
 .buildkite/steps/upload-release-steps.sh |  9 ---
 .buildkite/steps/upload-to-s3.sh         | 11 +--
 Dockerfile                               |  3 +
 9 files changed, 157 insertions(+), 29 deletions(-)
 create mode 100644 .buildkite/Dockerfile.public
 create mode 100644 .buildkite/steps/release-docker.sh
 delete mode 100755 .buildkite/steps/upload-release-steps.sh

diff --git a/.buildkite/Dockerfile.public b/.buildkite/Dockerfile.public
new file mode 100644
index 00000000..129ac39c
--- /dev/null
+++ b/.buildkite/Dockerfile.public
@@ -0,0 +1,6 @@
+FROM public.ecr.aws/docker/library/alpine:3.20.3@sha256:1e42bbe2508154c9126d48c2b8a75420c3544343bf86fd041fb7527e017a4b4a
+ARG TARGETARCH
+RUN apk update && apk add --no-cache curl ca-certificates
+COPY dist/buildkite-agent-metrics-linux-${TARGETARCH} ./buildkite-agent-metrics
+EXPOSE 8080 8125
+ENTRYPOINT ["./buildkite-agent-metrics"]
diff --git a/.buildkite/pipeline.release.yml b/.buildkite/pipeline.release.yml
index e74f75e3..97e070ce 100644
--- a/.buildkite/pipeline.release.yml
+++ b/.buildkite/pipeline.release.yml
@@ -12,7 +12,7 @@ steps:
     concurrency_group: "release_buildkite_metrics_github"
     plugins:
       - aws-assume-role-with-web-identity:
-          role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-buildkite-agent-metrics
+          role-arn: arn:aws:iam::172840064832:role/pipeline-buildkite-buildkite-agent-metrics-release
 
   - label: ":github: Release"
     command: ".buildkite/steps/release-github.sh"
@@ -22,7 +22,7 @@ steps:
       queue: "elastic-runners"
     plugins:
       - aws-assume-role-with-web-identity:
-          role-arn: arn:aws:iam::445615400570:role/pipeline-buildkite-buildkite-agent-metrics
+          role-arn: arn:aws:iam::172840064832:role/pipeline-buildkite-buildkite-agent-metrics-release
       - aws-ssm#v1.0.0:
           parameters:
             GITHUB_RELEASE_ACCESS_TOKEN: /pipelines/buildkite/buildkite-agent-metrics/GITHUB_RELEASE_ACCESS_TOKEN
@@ -31,3 +31,13 @@ steps:
           run: release
           cli-version: 2
           mount-buildkite-agent: true
+
+  - label: ":docker: Push"
+    command: ".buildkite/steps/release-docker.sh"
+    depends_on:
+      - block-release
+    agents:
+      queue: "elastic-runners"
+    plugins:
+      - aws-assume-role-with-web-identity:
+          role-arn: arn:aws:iam::172840064832:role/pipeline-buildkite-buildkite-agent-metrics-release
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 6094c42c..a81a9893 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,3 +1,6 @@
+env:
+  RELEASE_DRY_RUN: false
+
 steps:
   - name: ":test_tube: Test"
     key: test
@@ -60,9 +63,19 @@ steps:
       - aws-assume-role-with-web-identity:
           role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-buildkite-agent-metrics
 
-  - name: ":pipeline:"
-    key: upload-release-steps
+  - if: build.env("RELEASE_DRY_RUN") == "true" || build.env("BUILDKITE_TAG") =~ /^v\d+\.\d+\.\d+$$/
+    name: ":rocket: Release"
+    key: trigger-release
+    trigger: "buildkite-agent-metrics-release"
+    async: false
     depends_on:
       - build-binary
       - upload-to-s3
-    command: .buildkite/steps/upload-release-steps.sh
+    build:
+      message: "Release for ${BUILDKITE_TAG}, build ${BUILDKITE_BUILD_NUMBER}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      meta_data:
+        metrics-artifacts-build: "${BUILDKITE_BUILD_ID}"
+      env:
+        RELEASE_DRY_RUN: "${RELEASE_DRY_RUN:-false}"
diff --git a/.buildkite/steps/release-docker.sh b/.buildkite/steps/release-docker.sh
new file mode 100644
index 00000000..49855f69
--- /dev/null
+++ b/.buildkite/steps/release-docker.sh
@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+
+set -Eeufo pipefail
+
+source .buildkite/lib/release_dry_run.sh
+
+if [[ "${RELEASE_DRY_RUN:-false}" != "true" && "${BUILDKITE_BRANCH}" != "${BUILDKITE_TAG:-}" ]]; then
+  echo "Skipping release for a non-tag build on ${BUILDKITE_BRANCH}" >&2
+  exit 0
+fi
+
+registry="public.ecr.aws/buildkite/agent-metrics"
+image_tag="${registry}:build-${BUILDKITE_BUILD_NUMBER}"
+
+echo --- Fetching tags
+git fetch --prune --force origin "+refs/tags/*:refs/tags/*"
+
+echo --- Checking tags
+version="$(awk -F\" '/const Version/ {print $2}' version/version.go)"
+tag="v${version#v}"
+
+if [[ "${RELEASE_DRY_RUN:-false}" != true && "${tag}" != "${BUILDKITE_TAG}" ]]; then
+  echo "Error: version.go has not been updated to ${BUILDKITE_TAG#v}"
+  exit 1
+fi
+
+echo --- Downloading binaries
+
+artifacts_build="$(buildkite-agent meta-data get "metrics-artifacts-build")"
+
+rm -rf dist
+mkdir -p dist
+buildkite-agent artifact download --build "${artifacts_build}" "dist/buildkite-agent-metrics-linux-*" ./dist
+
+echo "--- Building :docker: image"
+
+builder_name="$(docker buildx create --use)"
+# shellcheck disable=SC2064 # we want the current $builder_name to be trapped, not the runtime one
+trap "docker buildx rm ${builder_name} || true" EXIT
+
+echo "--- Building :docker: ${image_tag} for all architectures"
+docker buildx build \
+  --progress plain \
+  --builder "${builder_name}" \
+  --platform linux/amd64,linux/arm64 \
+  --file .buildkite/Dockerfile.public \
+  .
+
+# Tag images for just the native architecture. There is a limitation in docker that prevents this
+# from being done in one command. Luckliy the second build will be quick because of docker layer caching
+# As this is just a native build, we don't need the lock.
+docker buildx build \
+  --progress plain \
+  --builder "${builder_name}" \
+  --tag "${image_tag}" \
+  --file .buildkite/Dockerfile.public \
+  --load \
+  .
+
+echo --- :ecr: Pushing to ECR
+
+# Convert 2.3.2 into [ 2.3.2 2.3 2 ] or 3.0-beta.42 in [ 3.0-beta.42 3.0 3 ]
+parse_version() {
+  local v="$1"
+  IFS='.' read -r -a parts <<< "${v%-*}"
+
+  for idx in $(seq 1 ${#parts[*]}) ; do
+    sed -e 's/ /./g' <<< "${parts[@]:0:$idx}"
+  done
+
+  [[ "${v%-*}" == "${v}" ]] || echo "${v}"
+}
+
+version_tags=($(parse_version "${version#v}"))
+
+# Do another build with all architectures and tags. The layers should be cached
+# from the previous build with all architectures.
+# Pushing to the docker registry in this way greatly simplifies creating the
+# manifest list on the docker registry so that either architecture can be pulled
+# with the same tag.
+release_dry_run docker buildx build \
+    --progress plain \
+    --builder "${builder_name}" \
+    --tag "${registry}:latest" \
+    "${version_tags[@]/#/--tag ${registry}:v}" \
+    --platform linux/amd64,linux/arm64 \
+    --file .buildkite/Dockerfile.public \
+    --push \
+    .
diff --git a/.buildkite/steps/release-github.sh b/.buildkite/steps/release-github.sh
index bd4a7fc7..8a5adb64 100755
--- a/.buildkite/steps/release-github.sh
+++ b/.buildkite/steps/release-github.sh
@@ -4,7 +4,12 @@ set -eufo pipefail
 
 source .buildkite/lib/release_dry_run.sh
 
-if [[ "$GITHUB_RELEASE_ACCESS_TOKEN" == "" ]]; then
+if [[ "${RELEASE_DRY_RUN:-false}" != "true" && "${BUILDKITE_BRANCH}" != "${BUILDKITE_TAG:-}" ]]; then
+  echo "Skipping release for a non-tag build on ${BUILDKITE_BRANCH}" >&2
+  exit 0
+fi
+
+if [[ "${GITHUB_RELEASE_ACCESS_TOKEN}" == "" ]]; then
   echo "Error: Missing \$GITHUB_RELEASE_ACCESS_TOKEN" >&2
   exit 1
 fi
@@ -13,27 +18,30 @@ echo --- Fetching tags
 git fetch --prune --force origin "+refs/tags/*:refs/tags/*"
 
 echo --- Downloading binaries
+
+artifacts_build="$(buildkite-agent meta-data get "metrics-artifacts-build")"
+
 rm -rf dist
 mkdir -p dist
-buildkite-agent artifact download "dist/*" ./dist
+buildkite-agent artifact download --build "${artifacts_build}" "dist/*" ./dist
 
 echo --- Checking tags
-version=$(awk -F\" '/const Version/ {print $2}' version/version.go)
+version="$(awk -F\" '/const Version/ {print $2}' version/version.go)"
 tag="v${version#v}"
 
-if [[ "${RELEASE_DRY_RUN:-false}" != true && $tag != "$BUILDKITE_TAG" ]]; then
+if [[ "${RELEASE_DRY_RUN:-false}" != true && "${tag}" != "${BUILDKITE_TAG}" ]]; then
   echo "Error: version.go has not been updated to ${BUILDKITE_TAG#v}"
   exit 1
 fi
 
-last_tag=$(git describe --tags --abbrev=0 --exclude "$tag")
+last_tag=$(git describe --tags --abbrev=0 --exclude "${tag}")
 
 # escape . so we can use in regex
 escaped_tag="${tag//\./\\.}"
 escaped_last_tag="${last_tag//\./\\.}"
 
-if [[ "${RELEASE_DRY_RUN:-false}" != true ]] && ! grep "^## \[$escaped_tag\]" CHANGELOG.md; then
-  echo "Error: CHANGELOG.md has not been updated for $tag" >&2
+if [[ "${RELEASE_DRY_RUN:-false}" != true ]] && ! grep "^## \[${escaped_tag}\]" CHANGELOG.md; then
+  echo "Error: CHANGELOG.md has not been updated for ${tag}" >&2
   exit 1
 fi
 
@@ -52,16 +60,16 @@ echo --- The following notes will accompany the release:
 notes=$(sed -n "/^## \[${escaped_tag}\]/,/^## \[${escaped_last_tag}\]/p" CHANGELOG.md | sed '$d')
 
 echo --- The following notes will accompany the release:
-echo "$notes"
+echo "${notes}"
 
 echo --- :github: Publishing draft release
 # TODO: add the following flag once github-cli in alpine repo hits v2.27+
 # --verify-tag \
 set +f
-GITHUB_TOKEN="$GITHUB_RELEASE_ACCESS_TOKEN" \
+GITHUB_TOKEN="${GITHUB_RELEASE_ACCESS_TOKEN}" \
   release_dry_run gh release create \
     --draft \
-    --notes "$notes" \
-    "$tag" \
+    --notes "${notes}" \
+    "${tag}" \
     dist/*
 set -f
diff --git a/.buildkite/steps/release-s3-version.sh b/.buildkite/steps/release-s3-version.sh
index b76d0f62..73316043 100755
--- a/.buildkite/steps/release-s3-version.sh
+++ b/.buildkite/steps/release-s3-version.sh
@@ -3,4 +3,9 @@ set -euo pipefail
 
 source .buildkite/lib/release_dry_run.sh
 
+if [[ "${RELEASE_DRY_RUN:-false}" != "true" && "${BUILDKITE_BRANCH}" != "${BUILDKITE_TAG:-}" ]]; then
+  echo "Skipping release for a non-tag build on ${BUILDKITE_BRANCH}" >&2
+  exit 0
+fi
+
 release_dry_run .buildkite/steps/upload-to-s3.sh release
diff --git a/.buildkite/steps/upload-release-steps.sh b/.buildkite/steps/upload-release-steps.sh
deleted file mode 100755
index e85ea708..00000000
--- a/.buildkite/steps/upload-release-steps.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-if [[ "${RELEASE_DRY_RUN:-false}" != "true" && "$BUILDKITE_BRANCH" != "${BUILDKITE_TAG:-}" ]]; then
-  echo "Skipping release for a non-tag build on $BUILDKITE_BRANCH" >&2
-  exit 0
-fi
-
-buildkite-agent pipeline upload .buildkite/pipeline.release.yml
diff --git a/.buildkite/steps/upload-to-s3.sh b/.buildkite/steps/upload-to-s3.sh
index b9a7d95b..d56797a7 100755
--- a/.buildkite/steps/upload-to-s3.sh
+++ b/.buildkite/steps/upload-to-s3.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -eu
 
-export AWS_DEFAULT_REGION=us-east-1
+export AWS_DEFAULT_REGION="us-east-1"
 
 # Updated from https://docs.aws.amazon.com/general/latest/gr/rande.html#lambda_region
 EXTRA_REGIONS=(
@@ -26,8 +26,8 @@ EXTRA_REGIONS=(
   sa-east-1
 )
 
-VERSION=$(awk -F\" '/const Version/ {print $2}' version/version.go)
-BASE_BUCKET=buildkite-lambdas
+VERSION="$(awk -F\" '/const Version/ {print $2}' version/version.go)"
+BASE_BUCKET="buildkite-lambdas"
 BUCKET_PATH="buildkite-agent-metrics"
 
 if [[ "${1:-}" == "release" ]] ; then
@@ -37,8 +37,11 @@ else
 fi
 
 echo "~~~ :buildkite: Downloading artifacts"
+
+artifacts_build="$(buildkite-agent meta-data get "metrics-artifacts-build")"
+
 mkdir -p dist
-buildkite-agent artifact download dist/handler.zip ./dist
+buildkite-agent artifact download --build "${artifacts_build}" dist/handler.zip ./dist
 
 echo "--- :s3: Uploading lambda to ${BASE_BUCKET}/${BUCKET_PATH}/ in ${AWS_DEFAULT_REGION}"
 aws s3 cp --acl public-read dist/handler.zip "s3://${BASE_BUCKET}/${BUCKET_PATH}/handler.zip"
diff --git a/Dockerfile b/Dockerfile
index a71f0c59..1cde1574 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,6 @@
+# Use this Dockerfile when building the image yourself.
+# The Buildkite pipeline is similar, but copies prebuilt binaries instead of
+# rebuilding them.
 FROM public.ecr.aws/docker/library/golang:1.23.3@sha256:73f06be4578c9987ce560087e2e2ea6485fb605e3910542cadd8fa09fc5f3e31 AS builder
 WORKDIR /go/src/github.com/buildkite/buildkite-agent-metrics/
 COPY . .

From 896b3e9d0ed565f151b4502ac471cf12e224d0a2 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 4 Dec 2024 16:16:22 +1100
Subject: [PATCH 2/3] Partially revert assume role changes

Co-authored-by: James Healy <james@yob.id.au>
---
 .buildkite/pipeline.release.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.release.yml b/.buildkite/pipeline.release.yml
index 97e070ce..bb25eaeb 100644
--- a/.buildkite/pipeline.release.yml
+++ b/.buildkite/pipeline.release.yml
@@ -12,7 +12,7 @@ steps:
     concurrency_group: "release_buildkite_metrics_github"
     plugins:
       - aws-assume-role-with-web-identity:
-          role-arn: arn:aws:iam::172840064832:role/pipeline-buildkite-buildkite-agent-metrics-release
+          role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-buildkite-agent-metrics
 
   - label: ":github: Release"
     command: ".buildkite/steps/release-github.sh"
@@ -22,7 +22,7 @@ steps:
       queue: "elastic-runners"
     plugins:
       - aws-assume-role-with-web-identity:
-          role-arn: arn:aws:iam::172840064832:role/pipeline-buildkite-buildkite-agent-metrics-release
+          role-arn: arn:aws:iam::445615400570:role/pipeline-buildkite-buildkite-agent-metrics-release
       - aws-ssm#v1.0.0:
           parameters:
             GITHUB_RELEASE_ACCESS_TOKEN: /pipelines/buildkite/buildkite-agent-metrics/GITHUB_RELEASE_ACCESS_TOKEN

From 26206f2822e786736601398856822bb9b417c88b Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Wed, 4 Dec 2024 16:33:06 +1100
Subject: [PATCH 3/3] Use new GH token path

Co-authored-by: James Healy <james@yob.id.au>
---
 .buildkite/pipeline.release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.release.yml b/.buildkite/pipeline.release.yml
index bb25eaeb..791e69ab 100644
--- a/.buildkite/pipeline.release.yml
+++ b/.buildkite/pipeline.release.yml
@@ -25,7 +25,7 @@ steps:
           role-arn: arn:aws:iam::445615400570:role/pipeline-buildkite-buildkite-agent-metrics-release
       - aws-ssm#v1.0.0:
           parameters:
-            GITHUB_RELEASE_ACCESS_TOKEN: /pipelines/buildkite/buildkite-agent-metrics/GITHUB_RELEASE_ACCESS_TOKEN
+            GITHUB_RELEASE_ACCESS_TOKEN: /pipelines/buildkite/buildkite-agent-metrics-release/GITHUB_RELEASE_ACCESS_TOKEN
       - docker-compose#v5.5.0:
           config: .buildkite/docker-compose.yaml
           run: release