Merge branch 'master' into rfc/katrina-offloaded-literal

flyteorg · Aug 13, 2024 · e89f434 · e89f434
2 parents 736e338 + 37b4e13
commit e89f434
Show file tree

Hide file tree

Showing 1,403 changed files with 95,717 additions and 33,576 deletions.
diff --git a/.codespellrc b/.codespellrc
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,11 +1,13 @@
 ## Tracking issue
-_https://github.com/flyteorg/flyte/issues/<number>_
-
-<!-- If your PR fixes an open issue, use `Closes #999` to link your PR with the issue. #999 stands for the issue number you are fixing -->
+<!--
+If your PR fixes an open issue, use `Closes #999` to link your PR with the issue.
+Example: Closes #999
 
-<!-- Remove this section if not applicable -->
+If your PR is related to an issue or PR, use `Related to #999` to link your PR.
+Example: Related to #999
 
-<!-- Example: Closes #31 -->
+Remove this section if not applicable
+-->
 
 ## Why are the changes needed?
 

diff --git a/.github/actions/clear-action-cache/action.yml b/.github/actions/clear-action-cache/action.yml
@@ -0,0 +1,11 @@
+name: 'Clear action cache'
+description: 'As suggested by GitHub to prevent low disk space: https://github.com/actions/runner-images/issues/2840#issuecomment-790492173'
+runs:
+  using: 'composite'
+  steps:
+    - shell: bash
+      run: |
+        rm -rf /usr/share/dotnet
+        rm -rf /opt/ghc
+        rm -rf "/usr/local/share/boost"
+        rm -rf "$AGENT_TOOLSDIRECTORY"
diff --git a/.github/codespell-ignored-words b/.github/codespell-ignored-words
@@ -0,0 +1,11 @@
+ThirdParty
+bootup
+crate
+fo
+lightyear
+nd
+notin
+querys
+ser
+te
+updAt
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -1,7 +1,7 @@
 name: Components Checks
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 
 on:
@@ -10,7 +10,7 @@ on:
     branches:
       - master
 env:
-  GO_VERSION: "1.21"
+  GO_VERSION: "1.22"
   PRIORITIES: "P0"
 jobs:
   unpack-envvars:
@@ -34,6 +34,7 @@ jobs:
           - flyteadmin
           # TODO(monorepo): Enable lint flytecopilot
           # - flytecopilot
+          - flytectl
           - flyteidl
           - flyteplugins
           - flytepropeller
@@ -53,6 +54,7 @@ jobs:
           - datacatalog
           - flyteadmin
           - flytecopilot
+          - flytectl
           - flyteplugins
           - flytepropeller
           - flytestdlib
@@ -61,35 +63,20 @@ jobs:
       component: ${{ matrix.component }}
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
     secrets:
-      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
-  docker-build:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  flyteadmin-int-tests-image-build:
     strategy:
       fail-fast: false
-      matrix:
-        component:
-          - datacatalog
-          - flyteadmin
-          - flytecopilot
-          - flytepropeller
-    name: Docker Build Images
+    name: flyteadmin integration tests image 
     uses: ./.github/workflows/component_docker_build.yml
     with:
-       component: ${{ matrix.component }}
+      component: flyteadmin
 
-  # TODO(monorepo): these tests are broken. They never test an actual change.
-  # endtoend:
-  #   name: End2End Test
-  #   needs: [ docker-build ]
-  #   uses: ./.github/workflows/end2end.yml
-  #   with:
-  #     # Reusing the output of the matrix is ok as they are essentially writing the same value (i.e. the directory artifacts are written + run id)
-  #     cache_key: ${{ needs.docker-build.outputs.cache_key }}
-  #     priorities: "P0"
-
-  integration:
+  flyteadmin-integration-tests:
     name: Integration Test
     needs:
-      - docker-build
+      - flyteadmin-int-tests-image-build
       - unpack-envvars
     strategy:
       fail-fast: false
@@ -99,7 +86,7 @@ jobs:
     uses: ./.github/workflows/integration.yml
     with:
       component: ${{ matrix.component }}
-      cache_key: ${{ needs.docker-build.outputs.cache_key }}
+      cache_key: ${{ needs.flyteadmin-int-tests-image-build.outputs.cache_key }}
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
 
   generate:
@@ -113,6 +100,7 @@ jobs:
           - datacatalog
           - flyteadmin
           - flytecopilot
+          - flytectl
           - flytepropeller
     uses: ./.github/workflows/go_generate.yml
     with:
@@ -129,3 +117,59 @@ jobs:
     secrets:
       FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
       FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}
+
+  dry_run_goreleaser:
+    name: Dry Run Goreleaser
+    runs-on: ubuntu-latest
+    needs:
+      - unpack-envvars
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: "0"
+      - uses: actions/setup-go@v4
+        with:
+          go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+      - name: Run GoReleaser dry run
+        uses: goreleaser/goreleaser-action@v5
+        with:
+          distribution: goreleaser-pro
+          version: latest
+          args: --snapshot  --skip-publish --rm-dist -f flytectl/.goreleaser.yml
+        env:
+          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
+
+  sandbox:
+    name: Test Getting started
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: flytectl
+    needs:
+      - unpack-envvars
+    steps:
+      - uses: insightsengineering/disk-space-reclaimer@v1
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+      - name: Build Flytectl binary
+        run: make compile
+      - name: Create a sandbox cluster
+        run: |
+          bin/flytectl demo start
+      - name: Setup flytectl config
+        run: bin/flytectl config init
+      - name: Register cookbook
+        run: bin/flytectl register examples -d development  -p flytesnacks
+      - name: Teardown Sandbox cluster
+        run: bin/flytectl demo teardown
diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml
@@ -2,7 +2,7 @@
 name: Codespell
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 
 on:
@@ -21,8 +21,9 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Codespell
         uses: codespell-project/actions-codespell@v2
         with:
-          skip: "*.pb,monodocs-environment.lock.yaml"
+          skip: "*.pb,monodocs-environment.lock.yaml,.git,*.pdf,*.svg,go.sum,go.mod,*requirements.txt,gen"
+          ignore_words_file: .github/codespell-ignored-words
diff --git a/.github/workflows/component_docker_build.yml b/.github/workflows/component_docker_build.yml
@@ -17,7 +17,7 @@ jobs:
       cache_key: ${{ steps.cache_key.outputs.cache_key }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - id: load-docker-cache
         name: Load Docker Cache
         uses: actions/cache@v3

diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
@@ -11,7 +11,7 @@ jobs:
     name: Generate git tags
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '0'
       - uses: actions/github-script@v6
@@ -28,7 +28,6 @@ jobs:
               "datacatalog",
               "flyteadmin",
               "flytecopilot",
-              "flyteidl",
               "flyteplugins",
               "flytepropeller",
               "flytestdlib",
@@ -60,7 +59,7 @@ jobs:
       - generate-tags
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: "0"
 
@@ -88,14 +87,15 @@ jobs:
           [
               datacatalog,
               flyteadmin,
+              flyteagent,
               flyteconsole,
               flytecopilot,
               flytepropeller,
               flytescheduler,
           ]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: "0"
 
@@ -107,6 +107,8 @@ jobs:
         run: |
           if [ ${{ matrix.component }} = "flytecopilot" ]; then
              echo ::set-output name=version::$(yq eval '.configmap.copilot.plugins.k8s.co-pilot.image' charts/flyte-core/values.yaml | cut -d ":" -f 2 )
+          elif [ ${{ matrix.component }} = "flyteagent" ]; then
+             echo ::set-output name=version::$(yq eval '.image.tag' charts/flyteagent/values.yaml)
           else
              echo ::set-output name=version::$(yq eval '.${{ matrix.component }}.image.tag' charts/flyte-core/values.yaml)
           fi
@@ -132,7 +134,7 @@ jobs:
       - build-docker-images
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: "0"
 
@@ -165,7 +167,7 @@ jobs:
       - build-docker-images
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: "0"