Merge pull request #287 from flyte-v1.12.0

Signed-off-by: Andrew Dye <[email protected]>

.gitattributes

@@ -1 +1,5 @@
 docs/**/*html linguist-generated=true
+flyteidl/gen/** linguist-generated=true
+flyteidl/protos/**/*.rst linguist-generated=true
+flyteidl/clients/go/assets/admin.swagger.json linguist-generated=true
+

.github/codespell-ignored-words

@@ -0,0 +1,11 @@
+ThirdParty
+bootup
+crate
+fo
+lightyear
+nd
+notin
+querys
+ser
+te
+updAt

.github/workflows/checks.yml

@@ -36,6 +36,7 @@ jobs:
           - flyteadmin
           # TODO(monorepo): Enable lint flytecopilot
           # - flytecopilot
+          - flytectl
           - flyteidl
           - flyteplugins
           - flytepropeller
@@ -57,6 +58,7 @@ jobs:
           - fasttask/plugin
           - flyteadmin
           - flytecopilot
+          - flytectl
           - flyteplugins
           - flytepropeller
           - flytestdlib
@@ -77,7 +79,7 @@ jobs:
     name: Docker Build Images
     uses: ./.github/workflows/component_docker_build.yml
     with:
-       component: ${{ matrix.component }}
+      component: ${{ matrix.component }}
 
   # TODO(monorepo): these tests are broken. They never test an actual change.
   # endtoend:
@@ -118,6 +120,7 @@ jobs:
           - fasttask/plugin
           - flyteadmin
           - flytecopilot
+          - flytectl
           - flytepropeller
     uses: ./.github/workflows/go_generate.yml
     with:
@@ -133,3 +136,59 @@ jobs:
   build_and_test_single_binary:
     name: Build and Test Single Binary Image
     uses: ./.github/workflows/build-and-test-single-binary-image.yml
+
+  dry_run_goreleaser:
+    name: Dry Run Goreleaser
+    runs-on: ubuntu-latest
+    needs:
+      - unpack-envvars
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: "0"
+      - uses: actions/setup-go@v4
+        with:
+          go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+      - name: Run GoReleaser dry run
+        uses: goreleaser/goreleaser-action@v5
+        with:
+          distribution: goreleaser-pro
+          version: latest
+          args: --snapshot  --skip-publish --rm-dist -f flytectl/.goreleaser.yml
+        env:
+          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
+
+  sandbox:
+    name: Test Getting started
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: flytectl
+    needs:
+      - unpack-envvars
+    steps:
+      - uses: insightsengineering/disk-space-reclaimer@v1
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+      - name: Build Flytectl binary
+        run: make compile
+      - name: Create a sandbox cluster
+        run: |
+          bin/flytectl demo start
+      - name: Setup flytectl config
+        run: bin/flytectl config init
+      - name: Register cookbook
+        run: bin/flytectl register examples -d development  -p flytesnacks
+      - name: Teardown Sandbox cluster
+        run: bin/flytectl demo teardown

.github/workflows/codespell.yml

@@ -21,8 +21,9 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Codespell
         uses: codespell-project/actions-codespell@v2
         with:
-          skip: "*.pb,monodocs-environment.lock.yaml"
+          skip: "*.pb,monodocs-environment.lock.yaml,.git,*.pdf,*.svg,go.sum,go.mod,*requirements.txt,gen"
+          ignore_words_file: .github/codespell-ignored-words

.github/workflows/component_docker_build.yml

@@ -17,7 +17,7 @@ jobs:
       cache_key: ${{ steps.cache_key.outputs.cache_key }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - id: load-docker-cache
         name: Load Docker Cache
         uses: actions/cache@v3

.github/workflows/end2end.yml

@@ -24,7 +24,7 @@ jobs:
           FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)"
           echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV}
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - uses: unionai/[email protected]
         name: Setup flytectl
       - uses: actions/setup-python@v3
@@ -78,7 +78,7 @@ jobs:
           pip freeze
       - name: Checkout flytesnacks
         if: ${{ inputs.priorities == 'P0' }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: flyteorg/flytesnacks
           path: flytesnacks

.github/workflows/go_generate.yml

@@ -17,10 +17,10 @@ jobs:
       run:
         working-directory: ${{ inputs.component }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: arduino/setup-protoc@v1
       - uses: bufbuild/buf-setup-action@v1
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
           go-version: ${{ inputs.go-version }}
       - name: Go generate and diff

.github/workflows/integration.yml

@@ -22,7 +22,9 @@ jobs:
         working-directory: ${{ inputs.component }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: "0"
       - id: load-docker-cache
         name: Load Docker Cache
         uses: actions/cache@v3
@@ -35,7 +37,7 @@ jobs:
         with:
           version: "v0.11.1"
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ inputs.go-version }}
       - name: Integration

.github/workflows/lint.yml

@@ -18,9 +18,9 @@ jobs:
         working-directory: ${{ inputs.component }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ inputs.go-version }}
       - name: Lint

.github/workflows/tests.yml

@@ -14,31 +14,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Fetch the code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v4
         with:
           go-version: "1.21"
       - name: Compile
         run: make compile
       - name: Run tests
         run: make test_unit
 
-  generate_kustomize:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-        with:
-          go-version: "1.21"
-      - name: Kustomize and diff
-        run: DELTA_CHECK=true make kustomize
-
   generate_helm:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
         with:
           go-version: "1.21"
       - name: Helm and diff

.github/workflows/unit-tests.yml

@@ -18,9 +18,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ inputs.go-version }}
       - name: Unit Tests

.github/workflows/validate-helm-charts.yaml

@@ -0,0 +1,89 @@
+name: Validate helm charts & manifests
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - "charts/**"
+      - "deployment/**"
+      - "docker/sandbox-bundled/manifests/**"
+
+jobs:
+  lint-and-test-charts:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          sparse-checkout: charts
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          check-latest: true
+
+      - uses: unionai/[email protected]
+        name: Setup flytectl
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2
+
+      - name: Detect charts changed (list-changed)
+        id: charts-changed
+        run: |
+          changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }})
+          if [[ -n "$changed" ]]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Run chart-testing (lint)
+        if: steps.charts-changed.outputs.changed == 'true'
+        run: |
+          ct lint \
+            --target-branch ${{ github.event.repository.default_branch }} \
+            --validate-maintainers=false \
+            --check-version-increment=false \
+            --chart-repos spark-operator=https://kubeflow.github.io/spark-operator \
+            --chart-repos dask=https://helm.dask.org \
+            --chart-repos bitnami=https://charts.bitnami.com/bitnami \
+            --chart-repos twuni=https://helm.twun.io \
+            --chart-repos kubernetes-dashboard=https://kubernetes.github.io/dashboard
+
+  validate-manifests:
+    needs:
+      - lint-and-test-charts
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        k8s_versions: [ "1.29.2", "1.28.7", "1.27.11" ]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            deployment
+            docker/sandbox-bundled/manifests
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+
+      - name: Install kubeconform
+        run: |
+          curl -L -o kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.4/kubeconform-linux-amd64.tar.gz
+          tar -zvxf kubeconform.tar.gz
+          chmod +x kubeconform
+          sudo mv kubeconform /usr/local/bin/kubeconform
+
+      - name: Validate manifests
+        run: |
+          kubeconform -strict -summary -skip CustomResourceDefinition -ignore-filename-pattern "deployment/stats/prometheus/*" -kubernetes-version ${{ matrix.k8s_versions }} ./deployment ./docker/sandbox-bundled/manifests

.gitignore

@@ -2,6 +2,7 @@
 _build/
 _bin/
 build/
+.tmp_build/
 *.swp
 *.swo
 .DS_Store
@@ -19,8 +20,6 @@ __pycache__/
 /charts/flyteagent/Chart.lock
 /charts/flyte-deps/Chart.lock
 /charts/flyte-core/Chart.lock
-/docker/sandbox-lite/storage/charts
-/docker/sandbox-lite/storage/Chart.lock
 **/*.bak-
 .python-version
 dist/

.readthedocs.yml

@@ -10,6 +10,7 @@ build:
     python: "mambaforge-22.9"
   jobs:
     post_install:
+      - cat monodocs-environment.lock.yaml
       - conda-lock install --name $READTHEDOCS_VERSION monodocs-environment.lock.yaml
       - pip install ./flyteidl
       - conda info

.run/single-binary.run.xml

@@ -0,0 +1,13 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="single-binary" type="GoApplicationRunConfiguration" factoryName="Go Application">
+    <module name="flyte" />
+    <working_directory value="$PROJECT_DIR$" />
+    <go_parameters value="-tags console" />
+    <parameters value="start --config=flyte-single-binary-local.yaml" />
+    <kind value="PACKAGE" />
+    <package value="github.com/flyteorg/flyte/cmd" />
+    <directory value="$PROJECT_DIR$" />
+    <filePath value="$PROJECT_DIR$" />
+    <method v="2" />
+  </configuration>
+</component>

CHANGELOG/CHANGELOG-v0.11.0.md

@@ -4,7 +4,7 @@
 * New to flyte? https://start.flyte.org takes you through first run experience. (Thanks to @jeevb)
 * [Grafana templates](https://docs.flyte.org/en/latest/howto/monitoring/index.html) for monitoring Flyte System and User Workflows.
 * [Extend Flyte](https://docs.flyte.org/en/latest/plugins/index.html) docs.
-* [FlyteIdl Docs](https://docs.flyte.org/projects/flyteidl/en/latest/) are published! You can learn about the core language that makes it all work.
+* [FlyteIdl Docs](https://docs.flyte.org/en/latest/reference_flyteidl.html) are published! You can learn about the core language that makes it all work.
 * [Additional knob](https://github.com/flyteorg/flytepropeller/pull/219/files#diff-91657d6448dfbf87f4cecf126ad02bd668ea233edcf74e860ef4f54bdd4cb552R78) for fine tuning flyte propeller performance that speeds up executions drastically.
 * OidC support for Google Idp (And other OidC compliant Idps)
 * Various stabilization bugs.