initial merge

paulhauner · Sep 25, 2023 · 6441c52 · 6441c52
2 parents 36f863a + 0f05499
commit 6441c52
Show file tree

Hide file tree

Showing 170 changed files with 3,910 additions and 3,099 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -17,6 +17,8 @@ env:
     DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
     IMAGE_NAME: ${{ github.repository_owner}}/lighthouse
     LCLI_IMAGE_NAME: ${{ github.repository_owner }}/lcli
+    # Enable self-hosted runners for the sigp repo only.
+    SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/lighthouse' }}
 
 jobs:
     # Extract the VERSION which is either `latest` or `vX.Y.Z`, and the VERSION_SUFFIX
@@ -48,7 +50,8 @@ jobs:
             VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }}
     build-docker-single-arch:
         name: build-docker-${{ matrix.binary }}${{ matrix.features.version_suffix }}
-        runs-on: ubuntu-22.04
+        # Use self-hosted runners only on the sigp repo.
+        runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release"]') || 'ubuntu-22.04'  }}
         strategy:
             matrix:
                 binary: [aarch64,
@@ -64,14 +67,13 @@ jobs:
 
         needs: [extract-version]
         env:
-            # We need to enable experimental docker features in order to use `docker buildx`
-            DOCKER_CLI_EXPERIMENTAL: enabled
             VERSION: ${{ needs.extract-version.outputs.VERSION }}
             VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
             FEATURE_SUFFIX: ${{ matrix.features.version_suffix }}
         steps:
             - uses: actions/checkout@v3
             - name: Update Rust
+              if: env.SELF_HOSTED_RUNNERS == 'false'
               run: rustup update stable
             - name: Dockerhub login
               run: |
@@ -80,16 +82,14 @@ jobs:
               run: |
                   cargo install cross
                   env CROSS_PROFILE=${{ matrix.profile }} CROSS_FEATURES=${{ matrix.features.env }} make build-${{ matrix.binary }}
+            - name: Make bin dir
+              run: mkdir ./bin
             - name: Move cross-built binary into Docker scope (if ARM)
               if: startsWith(matrix.binary, 'aarch64')
-              run: |
-                  mkdir ./bin;
-                  mv ./target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin;
+              run: mv ./target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin
             - name: Move cross-built binary into Docker scope (if x86_64)
               if: startsWith(matrix.binary, 'x86_64')
-              run: |
-                  mkdir ./bin;
-                  mv ./target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin;
+              run: mv ./target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin
             - name: Map aarch64 to arm64 short arch
               if: startsWith(matrix.binary, 'aarch64')
               run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV
@@ -99,17 +99,24 @@ jobs:
             - name: Set modernity suffix
               if: endsWith(matrix.binary, '-portable') != true
               run: echo "MODERNITY_SUFFIX=-modern" >> $GITHUB_ENV;
-            # Install dependencies for emulation. Have to create a new builder to pick up emulation support.
-            - name: Build Dockerfile and push
-              run: |
-                  docker run --privileged --rm tonistiigi/binfmt --install ${SHORT_ARCH}
-                  docker buildx create --use --name cross-builder
-                  docker buildx build \
-                      --platform=linux/${SHORT_ARCH} \
-                      --file ./Dockerfile.cross . \
-                      --tag ${IMAGE_NAME}:${VERSION}-${SHORT_ARCH}${VERSION_SUFFIX}${MODERNITY_SUFFIX}${FEATURE_SUFFIX} \
-                      --provenance=false \
-                      --push
+
+            - name: Install QEMU
+              if: env.SELF_HOSTED_RUNNERS == 'false'
+              run: sudo apt-get update && sudo apt-get install -y qemu-user-static
+
+            - name: Set up Docker Buildx
+              if: env.SELF_HOSTED_RUNNERS == 'false'
+              uses: docker/setup-buildx-action@v2
+
+            - name: Build and push
+              uses: docker/build-push-action@v4
+              with:
+                file: ./Dockerfile.cross
+                context: .
+                platforms: linux/${{ env.SHORT_ARCH }}
+                push: true
+                tags: ${{ env.IMAGE_NAME }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}${{ env.MODERNITY_SUFFIX }}${{ env.FEATURE_SUFFIX }}
+
     build-docker-multiarch:
         name: build-docker-multiarch${{ matrix.modernity }}
         runs-on: ubuntu-22.04
@@ -118,20 +125,22 @@ jobs:
             matrix:
                 modernity: ["", "-modern"]
         env:
-            # We need to enable experimental docker features in order to use `docker manifest`
-            DOCKER_CLI_EXPERIMENTAL: enabled
             VERSION: ${{ needs.extract-version.outputs.VERSION }}
             VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
         steps:
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v2
+
             - name: Dockerhub login
               run: |
                   echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+
             - name: Create and push multiarch manifest
               run: |
-                  docker manifest create ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }} \
-                      --amend ${IMAGE_NAME}:${VERSION}-arm64${VERSION_SUFFIX}${{ matrix.modernity }} \
-                      --amend ${IMAGE_NAME}:${VERSION}-amd64${VERSION_SUFFIX}${{ matrix.modernity }};
-                  docker manifest push ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }}
+                  docker buildx imagetools create -t ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }} \
+                      ${IMAGE_NAME}:${VERSION}-arm64${VERSION_SUFFIX}${{ matrix.modernity }} \
+                      ${IMAGE_NAME}:${VERSION}-amd64${VERSION_SUFFIX}${{ matrix.modernity }};
+
     build-docker-lcli:
         runs-on: ubuntu-22.04
         needs: [extract-version]

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -14,6 +14,8 @@ env:
     DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
     REPO_NAME: ${{ github.repository_owner }}/lighthouse
     IMAGE_NAME: ${{ github.repository_owner }}/lighthouse
+    # Enable self-hosted runners for the sigp repo only.
+    SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/lighthouse' }}
 
 jobs:
     extract-version:
@@ -38,44 +40,45 @@ jobs:
                        x86_64-windows-portable]
                 include:
                     -   arch: aarch64-unknown-linux-gnu
-                        platform: ubuntu-latest
+                        runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest'  }}
                         profile: maxperf
                     -   arch: aarch64-unknown-linux-gnu-portable
-                        platform: ubuntu-latest
+                        runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest'  }}
                         profile: maxperf
                     -   arch: x86_64-unknown-linux-gnu
-                        platform: ubuntu-latest
+                        runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest'  }}
                         profile: maxperf
                     -   arch: x86_64-unknown-linux-gnu-portable
-                        platform: ubuntu-latest
+                        runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest'  }}
                         profile: maxperf
                     -   arch: x86_64-apple-darwin
-                        platform: macos-latest
+                        runner: macos-latest
                         profile: maxperf
                     -   arch: x86_64-apple-darwin-portable
-                        platform: macos-latest
+                        runner: macos-latest
                         profile: maxperf
                     -   arch: x86_64-windows
-                        platform: windows-2019
+                        runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows", "release"]') || 'windows-2019'  }}
                         profile: maxperf
                     -   arch: x86_64-windows-portable
-                        platform: windows-2019
+                        runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows", "release"]') || 'windows-2019'  }}
                         profile: maxperf
 
-        runs-on:    ${{ matrix.platform }}
+        runs-on:    ${{ matrix.runner }}
         needs: extract-version
         steps:
             - name: Checkout sources
               uses: actions/checkout@v3
             - name: Get latest version of stable Rust
+              if: env.SELF_HOSTED_RUNNERS == 'false'
               run: rustup update stable
 
             # ==============================
             # Windows dependencies
             # ==============================
 
             - uses: KyleMayes/install-llvm-action@v1
-              if: startsWith(matrix.arch, 'x86_64-windows')
+              if: env.SELF_HOSTED_RUNNERS == 'false' && startsWith(matrix.arch, 'x86_64-windows')
               with:
                 version: "15.0"
                 directory: ${{ runner.temp }}/llvm

diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
@@ -58,7 +58,7 @@ jobs:
   release-tests-ubuntu:
     name: release-tests-ubuntu
     # Use self-hosted runners only on the sigp repo.
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "large"]') || 'ubuntu-latest'  }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest'  }}
     needs: cargo-fmt
     steps:
     - uses: actions/checkout@v3
@@ -73,21 +73,13 @@ jobs:
       run: make test-release
   release-tests-windows:
     name: release-tests-windows
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows"]') || 'windows-2019'  }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows", "CI"]') || 'windows-2019'  }}
     needs: cargo-fmt
     steps:
     - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       if: env.SELF_HOSTED_RUNNERS == false
       run: rustup update stable
-    - name: Use Node.js
-      uses: actions/setup-node@v2
-      with:
-        node-version: '14'
-    - name: Install windows build tools
-      run: |
-        choco install python visualstudio2019-workload-vctools -y
-        npm config set msvs_version 2019
     - name: Install Foundry (anvil)
       uses: foundry-rs/foundry-toolchain@v1
       with:
@@ -106,7 +98,7 @@ jobs:
   beacon-chain-tests:
     name: beacon-chain-tests
     # Use self-hosted runners only on the sigp repo.
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "large"]') || 'ubuntu-latest'  }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest'  }}
     needs: cargo-fmt
     steps:
     - uses: actions/checkout@v3
@@ -138,7 +130,7 @@ jobs:
   debug-tests-ubuntu:
     name: debug-tests-ubuntu
     # Use self-hosted runners only on the sigp repo.
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "large"]') || 'ubuntu-latest'  }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest'  }}
     needs: cargo-fmt
     steps:
     - uses: actions/checkout@v3
@@ -164,7 +156,7 @@ jobs:
   ef-tests-ubuntu:
     name: ef-tests-ubuntu
     # Use self-hosted runners only on the sigp repo.
-    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "small"]') || 'ubuntu-latest'  }}
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "small"]') || 'ubuntu-latest'  }}
     needs: cargo-fmt
     steps:
     - uses: actions/checkout@v3