Merge branch 'develop' into sync2/atxs

spacemeshos · Dec 5, 2024 · 9092fe7 · 9092fe7
2 parents 244ceb1 + 138e9bd
commit 9092fe7
Show file tree

Hide file tree

Showing 64 changed files with 1,957 additions and 1,204 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -86,8 +86,6 @@ jobs:
           make test-fmt
           make test-tidy
           make test-generate
-      - name: Check for vulnerabilities
-        run: make vulncheck
 
   lint:
     runs-on: ubuntu-22.04

diff --git a/.github/workflows/systest.yml b/.github/workflows/systest.yml
@@ -52,12 +52,12 @@ jobs:
             nondoc:
               - '!**/*.md'
 
-  systest:
+  build-docker-images:
     runs-on: ubuntu-22.04
     if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }}
     needs:
       - filter-changes
-    timeout-minutes: 70
+    timeout-minutes: 20
     permissions:
       contents: 'read'
       id-token: 'write'
@@ -66,35 +66,87 @@ jobs:
         with:
           ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }}
 
-      - name: Setup kubectl
-        id: install
-        uses: azure/setup-kubectl@v4
-        with:
-          version: "v1.27.16"
-
       - name: Setup gcloud authentication
         uses: google-github-actions/auth@v2
         with:
           project_id: ${{ secrets.GCP_WI_PROJECT_ID }}
           workload_identity_provider: ${{ secrets.GCP_WI_PROVIDER_SA }}
-          service_account: ${{ secrets.GCP_WI_SA }} 
+          service_account: ${{ secrets.GCP_WI_SA }}
           token_format: access_token
 
       - name: Configure gcloud
         uses: google-github-actions/setup-gcloud@v2
         with:
           version: "469.0.0"
 
-      - name: Configure gke authentication plugin
-        run: gcloud components install gke-gcloud-auth-plugin --quiet
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Configure kubectl
-        run: gcloud container clusters get-credentials ${{ secrets.CI_CLUSTER_NAME }} --region ${{ secrets.CI_REGION_NAME }} --project ${{ secrets.CI_GCP_PROJECT_ID }}
+      - uses: extractions/netrc@v2
+        with:
+          machine: github.com
+          username: ${{ secrets.GH_ACTION_TOKEN_USER }}
+          password: ${{ secrets.GH_ACTION_TOKEN }}
+        if: vars.GOPRIVATE
+
+      - name: Push go-spacemesh build to docker hub
+        run: make dockerpush
+
+      - name: Push go-bootstrapper build to docker hub
+        run: make dockerpush-bs
+
+      - name: Get commit hash
+        id: vars
+        shell: bash
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - uses: extractions/netrc@v2
+        with:
+          machine: github.com
+          username: ${{ secrets.GH_ACTION_TOKEN_USER }}
+          password: ${{ secrets.GH_ACTION_TOKEN }}
+        if: vars.GOPRIVATE
+
+      - name: Build tests docker image
+        run: make -C systest docker
+
+      - name: Push tests docker images
+        run: make -C systest push
+
+  provision-gke-cluster:
+    runs-on: ubuntu-22.04
+    if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }}
+    needs:
+      - filter-changes
+    timeout-minutes: 20
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }}
+
+      - name: Setup gcloud authentication
+        uses: google-github-actions/auth@v2
+        with:
+          project_id: ${{ secrets.GCP_WI_PROJECT_ID }}
+          workload_identity_provider: ${{ secrets.GCP_WI_PROVIDER_SA }}
+          service_account: ${{ secrets.GCP_WI_SA }}
+          token_format: access_token
+
+      - name: Configure gcloud
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          version: "469.0.0"
 
       - name: Create Node Pool
         run: |
-
           NODE_POOL_NAME="systemtest-${{ github.run_id }}"
+          echo "nodepool-name=${NODE_POOL_NAME}" >> $GITHUB_OUTPUT
 
           gcloud container node-pools create $NODE_POOL_NAME \
             --cluster ${{ secrets.CI_CLUSTER_NAME }} \
@@ -115,6 +167,47 @@ jobs:
 
           echo "Node pool created: $NODE_POOL_NAME"
 
+  systest-gke:
+    runs-on: ubuntu-22.04
+    if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }}
+    continue-on-error: true
+    needs:
+      - filter-changes
+      - provision-gke-cluster
+      - build-docker-images
+    timeout-minutes: 70
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }}
+
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v4
+        with:
+          version: "v1.27.16"
+
+      - name: Setup gcloud authentication
+        uses: google-github-actions/auth@v2
+        with:
+          project_id: ${{ secrets.GCP_WI_PROJECT_ID }}
+          workload_identity_provider: ${{ secrets.GCP_WI_PROVIDER_SA }}
+          service_account: ${{ secrets.GCP_WI_SA }}
+          token_format: access_token
+
+      - name: Configure gcloud
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          version: "469.0.0"
+
+      - name: Configure gke authentication plugin
+        run: gcloud components install gke-gcloud-auth-plugin --quiet
+
+      - name: Configure kubectl
+        run: gcloud container clusters get-credentials ${{ secrets.CI_CLUSTER_NAME }} --region ${{ secrets.CI_REGION_NAME }} --project ${{ secrets.CI_GCP_PROJECT_ID }}
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -128,12 +221,6 @@ jobs:
           password: ${{ secrets.GH_ACTION_TOKEN }}
         if: vars.GOPRIVATE
 
-      - name: Push go-spacemesh build to docker hub
-        run: make dockerpush
-
-      - name: Push go-bootstrapper build to docker hub
-        run: make dockerpush-bs
-
       - name: Get commit hash
         id: vars
         shell: bash
@@ -146,12 +233,6 @@ jobs:
           password: ${{ secrets.GH_ACTION_TOKEN }}
         if: vars.GOPRIVATE
 
-      - name: Build tests docker image
-        run: make -C systest docker
-
-      - name: Push tests docker images
-        run: make -C systest push
-
       - name: set up go
         uses: actions/setup-go@v5
         with:
@@ -175,9 +256,37 @@ jobs:
         env:
           test_id: systest-${{ steps.vars.outputs.sha_short }}
         run: make -C systest clean
-
+
+  delete-gke-cluster-pool:
+    runs-on: ubuntu-22.04
+    if: always()
+    needs:
+      - filter-changes
+      - provision-gke-cluster
+      - systest-gke
+    timeout-minutes: 20
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }}
+
+      - name: Setup gcloud authentication
+        uses: google-github-actions/auth@v2
+        with:
+          project_id: ${{ secrets.GCP_WI_PROJECT_ID }}
+          workload_identity_provider: ${{ secrets.GCP_WI_PROVIDER_SA }}
+          service_account: ${{ secrets.GCP_WI_SA }}
+          token_format: access_token
+
+      - name: Configure gcloud
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          version: "469.0.0"
+
       - name: Delete Node Pool
-        if: always()
         run: |
           NODE_POOL_NAME="systemtest-${{ github.run_id }}"
           gcloud container node-pools delete $NODE_POOL_NAME \
@@ -187,15 +296,92 @@ jobs:
             --quiet
           echo "Node pool deleted: $NODE_POOL_NAME"
 
+  systest-dedicated-cluster:
+    runs-on: ubuntu-22.04
+    if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }}
+    needs:
+      - filter-changes
+      - build-docker-images
+    timeout-minutes: 70
+    continue-on-error: true
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }}
+
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v4
+        with:
+          version: "v1.27.16"
+
+      - name: Save cluster kubeconfig
+        run: |
+          mkdir -p ${{ github.workspace }}/.kube
+          echo "${{ secrets.CI_KUBECONFIG }}" | base64 -d > ${{ github.workspace }}/.kube/config
+          echo "KUBECONFIG=${{ github.workspace }}/.kube/config" >> $GITHUB_ENV
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - uses: extractions/netrc@v2
+        with:
+          machine: github.com
+          username: ${{ secrets.GH_ACTION_TOKEN_USER }}
+          password: ${{ secrets.GH_ACTION_TOKEN }}
+        if: vars.GOPRIVATE
+
+      - name: Get commit hash
+        id: vars
+        shell: bash
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - uses: extractions/netrc@v2
+        with:
+          machine: github.com
+          username: ${{ secrets.GH_ACTION_TOKEN_USER }}
+          password: ${{ secrets.GH_ACTION_TOKEN }}
+        if: vars.GOPRIVATE
+
+      - name: set up go
+        uses: actions/setup-go@v5
+        with:
+          check-latest: true
+          go-version-file: "go.mod"
+
+      - name: Run tests
+        env:
+          test_id: systest-${{ steps.vars.outputs.sha_short }}
+          storage: default=10Gi
+          node_selector: cluster-type=baremetal
+          size: 20
+          bootstrap: 4m
+          level: ${{ inputs.log_level }}
+          clusters: 4
+          norbac: 1
+        run: make -C systest run test_name=${{ inputs.test_name }}
+
+      - name: Delete pod
+        if: always()
+        env:
+          test_id: systest-${{ steps.vars.outputs.sha_short }}
+        run: make -C systest clean
+
   systest-status:
     if: always()
     needs:
       - filter-changes
-      - systest
+      - systest-gke
+      - systest-dedicated-cluster
     runs-on: ubuntu-22.04
     env:
       # short-circuit success if no non-doc files were modified
-      status: ${{ (needs.filter-changes.outputs.nondocchanges == 'false' || needs.systest.result == 'success') && 'success' || 'failure' }}
+      status: ${{ (needs.filter-changes.outputs.nondocchanges == 'false' || needs.systest-gke.result == 'success' || needs.systest-dedicated-cluster.result == 'success') && 'success' || 'failure' }}
     steps:
       # print a single, clean status update to slack
       - uses: act10ns/slack@v2

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,27 @@ See [RELEASE](./RELEASE.md) for workflow instructions.
 
 ### Improvements
 
+* [#6457](https://github.com/spacemeshos/go-spacemesh/pull/6457) Fix trying to certify POST when certifier URL is empty.
+
+## v1.7.8
+
+### Upgrade information
+
+The deprecated `--poet-server` flag was removed from the node. Using the flag before this version would have resulted
+in an error asking to change to `--poet-servers`. Now the node will only print that the flag is unknown.
+Use `--poet-servers` instead.
+
+### Improvements
+
+* [#6477](https://github.com/spacemeshos/go-spacemesh/pull/6477) Improve peer selection during sync. A node will not
+  request data from nodes that have not yet completed their initialization.
+
+* [#6498](https://github.com/spacemeshos/go-spacemesh/pull/6498) Removed deprecated `--poet-server` flag. Use
+  `--poet-servers` instead.
+
+* [#6500](https://github.com/spacemeshos/go-spacemesh/pull/6500) Updated post-rs dependency. This fixes an issue where
+  the profiler would exit with an error when profiling PoS proving options.
+
 ## v1.7.7
 
 ### Improvements

diff --git a/Makefile b/Makefile
@@ -4,7 +4,6 @@ BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 
 GOLANGCI_LINT_VERSION := v1.61.0
 GOTESTSUM_VERSION := v1.12.0
-GOVULNCHECK_VERSION := v1.1.3
 GOSCALE_VERSION := v1.2.0
 MOCKGEN_VERSION := v0.5.0
 
@@ -69,7 +68,6 @@ install:
 	go install github.com/spacemeshos/go-scale/scalegen@$(GOSCALE_VERSION)
 	go install go.uber.org/mock/mockgen@$(MOCKGEN_VERSION)
 	go install gotest.tools/gotestsum@$(GOTESTSUM_VERSION)
-	go install golang.org/x/vuln/cmd/govulncheck@$(GOVULNCHECK_VERSION)
 .PHONY: install
 
 build: go-spacemesh get-profiler get-postrs-service
@@ -148,10 +146,6 @@ cover: get-libs
 	@$(ULIMIT) CGO_LDFLAGS="$(CGO_TEST_LDFLAGS)" gotestsum --junitfile junit.xml -- -coverprofile=cover.out -p 1 -timeout 30m -coverpkg=./... $(UNIT_TESTS)
 .PHONY: cover
 
-vulncheck: get-libs
-	govulncheck ./...
-.PHONY: vulncheck
-
 list-versions:
 	@echo "Latest 5 tagged versions:\n"
 	@git for-each-ref --sort=-creatordate --count=5 --format '%(creatordate:short): %(refname:short)' refs/tags

diff --git a/Makefile-libs.Inc b/Makefile-libs.Inc
@@ -46,7 +46,7 @@ else
 	endif
 endif
 
-POSTRS_SETUP_REV = 0.8.1
+POSTRS_SETUP_REV = 0.8.4
 POSTRS_SETUP_ZIP = libpost-$(platform)-v$(POSTRS_SETUP_REV).zip
 POSTRS_SETUP_URL_ZIP ?= https://github.com/spacemeshos/post-rs/releases/download/v$(POSTRS_SETUP_REV)/$(POSTRS_SETUP_ZIP)