Restructure releases (#4304)

Signed-off-by: Eduardo Apolinario <[email protected]>
Co-authored-by: Eduardo Apolinario <[email protected]>

.github/workflows/checks.yml

@@ -134,47 +134,11 @@ jobs:
       component: ${{ matrix.component }}
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
 
-  bump-tags:
-    name: Bump git tags
-    # TODO(monorepo): skip this if author is flyte-bot?
-    if: ${{ github.event_name != 'pull_request' }}
-    needs:
-      - integration
-      - lint
-      - unit-tests
-      - generate
-    uses: ./.github/workflows/bump-tags.yml
-    secrets:
-      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
-
-  # TODO(monorepo): we are not going to release any binary
-  # goreleaser:
-  #   name: Goreleaser
-  #   needs: [ bump_version ] # Only to ensure it can successfully build
-  #   uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master
-  #   with:
-  #     go-version: "1.19"
-  #   secrets:
-  #     FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
-
-  push_docker_image:
-    name: Build & Push Image
-    needs: [ bump-tags ]
-    strategy:
-      fail-fast: false
-      matrix:
-        component:
-          - datacatalog
-          - flyteadmin
-          - flytecopilot
-          - flytepropeller
-          - flytescheduler
-    uses: ./.github/workflows/publish.yml
+  build_docker_images:
+    name: Build Images
+    uses: ./.github/workflows/publish-images.yml
     with:
-      version: ${{ needs.bump-tags.outputs.next_version }}
-      component: ${{ matrix.component }}
-      dockerfile: Dockerfile.${{ matrix.component }}
-      push: true
+      push: false
     secrets:
       FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
       FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}

.github/workflows/create_release.yml

@@ -7,29 +7,57 @@ on:
         required: true
 
 jobs:
-  bump-version:
-    name: bump-version
+  generate-tags:
+    name: Generate git tags
     runs-on: ubuntu-latest
-    outputs:
-      version: ${{ steps.bump-version.outputs.tag }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
-          fetch-depth: "0"
+          fetch-depth: '0'
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.FLYTE_BOT_PAT }}
+          script: |
+            github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/tags/${{ github.event.inputs.version }}`,
+              sha: context.sha
+            })
+            const components = [
+              "datacatalog",
+              "flyteadmin",
+              "flytecopilot",
+              "flyteidl",
+              "flyteplugins",
+              "flytepropeller",
+              "flytestdlib",
+            ];
+            for (const c of components) {
+              github.rest.git.createRef({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                ref: `refs/tags/${c}/${{ github.event.inputs.version }}`,
+                sha: context.sha
+              })
+            }
 
-      - name: Bump version and push tag
-        id: bump-version
-        uses: anothrNick/[email protected]
-        env:
-          GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
-          WITH_V: true
-          CUSTOM_TAG: ${{ github.event.inputs.version }}
-          RELEASE_BRANCHES: master
+  build-docker-images:
+    needs:
+      - generate-tags
+    uses: ./.github/workflows/publish-images.yml
+    with:
+      version: ${{ github.event.inputs.version }}
+      push: true
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+      FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}
 
   publish-flyte-binary-image:
     name: Publish flyte binary image for the release version
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - generate-tags
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -45,24 +73,25 @@ jobs:
 
       - name: Tag image to release version
         run: |
-          for release in latest ${{ needs.bump-version.outputs.version }}; do
+          for release in latest ${{ github.event.inputs.version }}; do
             docker buildx imagetools create --tag "ghcr.io/${{ github.repository_owner }}/flyte-binary-release:${release}" "ghcr.io/${{ github.repository_owner }}/flyte-binary:sha-${{ github.sha }}"
           done
 
   publish-flyte-component-image:
     name: Publish flyte component image for the release version
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - build-docker-images
     strategy:
       matrix:
         component:
           [
+              datacatalog,
+              flyteadmin,
               flyteconsole,
+              flytecopilot,
               flytepropeller,
-              flyteadmin,
-              datacatalog,
               flytescheduler,
-              flytecopilot,
           ]
     steps:
       - name: Checkout
@@ -76,11 +105,7 @@ jobs:
       - name: Get Latest Version of component
         id: set_version
         run: |
-          if [ ${{ matrix.component }} = "flytecopilot" ]; then
-             echo ::set-output name=version::$(yq eval '.configmap.copilot.plugins.k8s.co-pilot.image' charts/flyte-core/values.yaml | cut -d ":" -f 2 )
-          else
-             echo ::set-output name=version::$(yq eval '.${{ matrix.component }}.image.tag' charts/flyte-core/values.yaml)
-          fi
+          echo ::set-output name=version::$(yq eval '.${{ matrix.component }}.image.tag' charts/flyte-core/values.yaml)
         shell: bash
 
       - name: Login to GitHub Container Registry
@@ -92,14 +117,15 @@ jobs:
 
       - name: Tag Image to release version
         run: |
-          for release in latest ${{ needs.bump-version.outputs.version }}; do
+          for release in latest ${{ github.event.inputs.version }}; do
             docker buildx imagetools create --tag "ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}-release:${release}" "ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}:${{ steps.set_version.outputs.version }}"
           done
 
   helm-release:
     name: Flyte helm release
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - build-docker-images
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -118,7 +144,7 @@ jobs:
           git config user.email "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
       - name: Prepare Flyte Helm Release
         env:
-          VERSION: ${{ needs.bump-version.outputs.version }}
+          VERSION: ${{ github.event.inputs.version }}
           REPOSITORY: "https://flyteorg.github.io/flyte"
         run: |
           make prepare_artifacts
@@ -131,7 +157,8 @@ jobs:
   manifest-release:
     name: Flyte manifest release
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - build-docker-images
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -140,7 +167,7 @@ jobs:
 
       - name: Prepare Flyte Release
         env:
-          VERSION: ${{ needs.bump-version.outputs.version }}
+          VERSION: ${{ github.event.inputs.version }}
         run: |
           make prepare_artifacts
           git stash

.github/workflows/generate_flyte_manifest.yml

@@ -2,19 +2,27 @@ name: Generate Flyte manifest
 
 on:
   workflow_dispatch:
+    inputs:
+      next-version:
+        description: "version name. example v0.1.1"
+        required: true
 
 jobs:
   update-flyte-releases:
     name: Update Flyte components
     runs-on: ubuntu-latest
+    needs:
+      - generate-tags
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: "0"
       - uses: actions/setup-go@v2
         with:
           go-version: "1.19"
-      - name: Build kustomize
+      - name: Update references
+        env:
+          VERSION: ${{ github.event.inputs.next-version }}
         run: |
           make release_automation
           make kustomize

.github/workflows/publish-images.yml

@@ -0,0 +1,41 @@
+name: Build & Push All Components Images
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        description: "Version of image"
+        required: false
+        type: string
+        default: ci-checks
+      push:
+        description: "Push to registry"
+        required: true
+        type: boolean
+    secrets:
+      FLYTE_BOT_USERNAME:
+        required: true
+      FLYTE_BOT_PAT:
+        required: true
+
+jobs:
+  push_docker_image:
+    name: Build & Push Image
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          - flytecopilot
+          - flytepropeller
+          - flytescheduler
+    uses: ./.github/workflows/publish.yml
+    with:
+      version: ${{ inputs.version }}
+      component: ${{ matrix.component }}
+      dockerfile: Dockerfile.${{ matrix.component }}
+      push: ${{ inputs.push }}
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+      FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}