[NU-1823] Kafka source without topic schema (#7066) #21634
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #this should be the same as workflow name in test-report.yml | |
| name: CI | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| - staging | |
| - demo | |
| - preview/* | |
| - release/* | |
| paths-ignore: | |
| - '**.md' | |
| - 'docs/**' | |
| push: | |
| branches: | |
| - master | |
| - staging | |
| - demo | |
| - preview/* | |
| - release/* | |
| paths-ignore: | |
| - '**.md' | |
| - 'docs/**' | |
| #TODO: currently release is done manually, we don't want to run this pipeline on released version, to avoid accidental pushes | |
| tags-ignore: | |
| - '**' | |
| env: | |
| #we use this variable in ciRunSbt.sh | |
| #NOTE: for publishing we use different settings, we don't use ciRunSbt.sh there | |
| CROSS_BUILD: ${{ github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/preview') || startsWith(github.ref, 'refs/heads/release') }} | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| fe_changes_count: ${{ steps.filter.outputs.fe_changes_count }} | |
| all_changes_count: ${{ steps.filter.outputs.all_changes_count }} | |
| git_source_branch: ${{ steps.variables.outputs.git_source_branch }} | |
| nk_snapshot_version: ${{ steps.variables.outputs.nk_snapshot_version }} | |
| scala_version_matrix: ${{ steps.variables.outputs.scala_version_matrix }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 100 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| list-files: 'csv' | |
| filters: | | |
| fe_changes: | |
| - 'designer/client/**' | |
| - 'designer/submodules/**' | |
| - 'docs/**' | |
| - '**.md' | |
| all_changes: | |
| - '**' | |
| - name: Define variables | |
| id: variables | |
| shell: bash | |
| run: | | |
| GIT_SOURCE_BRANCH=`([ "${GITHUB_HEAD_REF}" != "" ] && echo "${GITHUB_HEAD_REF}" || echo "${GITHUB_REF}") | sed 's/refs\/heads\///g'` | |
| SANITIZED_BRANCH=`echo ${GIT_SOURCE_BRANCH} | sed 's/[^a-zA-Z0-9._-]/\_/g' | awk '{print tolower($0)}'` | |
| VERSION_SUFFIX="-$SANITIZED_BRANCH-$(date -I)-$GITHUB_RUN_NUMBER-${GITHUB_SHA::9}" | |
| NK_SNAPSHOT_VERSION=`cat version.sbt | sed -e 's/.*:= *"//' -e 's/" *//' | sed "s/-SNAPSHOT/${VERSION_SUFFIX}-SNAPSHOT/"` | |
| echo "git_source_branch=$GIT_SOURCE_BRANCH" >> $GITHUB_OUTPUT | |
| echo "nk_snapshot_version=$NK_SNAPSHOT_VERSION" >> $GITHUB_OUTPUT | |
| if [[ $CROSS_BUILD == 'true' ]]; then | |
| echo "scala_version_matrix=[\"2.12\",\"2.13\"]" >> $GITHUB_OUTPUT | |
| else | |
| echo "scala_version_matrix=[\"2.13\"]" >> $GITHUB_OUTPUT | |
| fi | |
| build: | |
| name: Build | |
| runs-on: ubuntu-latest | |
| needs: [ setup ] | |
| env: | |
| # We can't just use conditional jobs mechanism ('if' directive) because 'cypressTests' job depends on this one | |
| shouldPerformBackendBuild: ${{ needs.setup.outputs.fe_changes_count != needs.setup.outputs.all_changes_count || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/demo' || startsWith(github.ref, 'refs/heads/preview') || startsWith(github.ref, 'refs/heads/release') }} | |
| steps: | |
| - name: Cancel previous runs | |
| if: ${{ env.shouldPerformBackendBuild == 'true' && github.event_name != 'push' }} | |
| uses: styfle/[email protected] | |
| with: | |
| access_token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/checkout@v3 | |
| if: ${{ env.shouldPerformBackendBuild == 'true' }} | |
| - uses: coursier/setup-action@v1 | |
| if: ${{ env.shouldPerformBackendBuild == 'true' }} | |
| with: | |
| jvm: temurin:1.11.0.17 | |
| - name: Cache ivy packages | |
| if: ${{ env.shouldPerformBackendBuild == 'true' }} | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.ivy2/cache | |
| ~/.cache/coursier | |
| ~/.sbt | |
| key: sbt-cache-${{ hashFiles('**/*.sbt') }} | |
| restore-keys: sbt | |
| - name: Build | |
| if: ${{ env.shouldPerformBackendBuild == 'true' }} | |
| shell: bash | |
| run: CROSS_BUILD=true ./ciRunSbt.sh Compile/compile Test/compile | |
| - name: Tar artifacts | |
| if: ${{ env.shouldPerformBackendBuild == 'true' }} | |
| shell: bash | |
| run: find . -wholename "**/target/**/*" -printf '%P\0' | tar --null -C '.' --files-from=- -czf 'target.tgz' | |
| - name: Store target | |
| if: ${{ env.shouldPerformBackendBuild == 'true' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-target | |
| path: target.tgz | |
| build-fe: | |
| name: BuildFrontend | |
| runs-on: ubuntu-latest | |
| needs: [ setup ] | |
| env: | |
| NUSSKNACKER_VERSION: ${{ needs.setup.outputs.nk_snapshot_version }} | |
| steps: | |
| - name: Cancel previous runs | |
| if: github.event_name != 'push' | |
| uses: styfle/[email protected] | |
| with: | |
| access_token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/checkout@v3 | |
| - name: Cache npm | |
| id: cache-npm | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| **/node_modules | |
| ~/.cache/Cypress | |
| key: ${{ hashFiles('designer/client/package-lock.json', 'designer/client/.nvmrc', 'designer/submodules/package-lock.json', 'designer/submodules/.nvmrc') }} | |
| - name: Get Node.js version | |
| id: nvm | |
| run: echo "NODE_VERSION=$(cat designer/client/.nvmrc)" >> $GITHUB_OUTPUT | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: ${{ steps.nvm.outputs.NODE_VERSION }} | |
| - name: Download node modules | |
| if: steps.cache-npm.outputs.cache-hit != 'true' | |
| run: | | |
| npm ci --no-audit --prefix designer/client | |
| npm ci --no-audit --prefix designer/submodules | |
| - name: Build FE | |
| run: (cd designer/client; npm run build) | |
| - name: Build FE submodules | |
| run: | | |
| cd designer | |
| cp -r client/.federated-types/nussknackerUi submodules/types/@remote | |
| cd submodules | |
| npm run build | |
| - name: Tar fe artifacts | |
| shell: bash | |
| run: tar -czf 'fe-dist.tgz' designer/client/dist designer/submodules/dist | |
| - name: Store dist | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-fe-dist | |
| path: fe-dist.tgz | |
| tests: | |
| name: Tests | |
| runs-on: ubuntu-latest | |
| needs: [ build, setup ] | |
| strategy: | |
| matrix: | |
| # Due to the different behaviour of Nussknacker's OpenAPI generation on different versions of scala we | |
| # have to run these tests always for every scala version. See NuDesignerApiAvailableToExposeYamlSpec | |
| scalaVersion: [2.12, 2.13] | |
| env: | |
| NUSSKNACKER_SCALA_VERSION: ${{ matrix.scalaVersion }} | |
| if: ${{ needs.setup.outputs.fe_changes_count != needs.setup.outputs.all_changes_count || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/demo' || startsWith(github.ref, 'refs/heads/preview') || startsWith(github.ref, 'refs/heads/release') }} | |
| steps: | |
| - name: Cancel previous runs | |
| if: github.event_name != 'push' | |
| uses: styfle/[email protected] | |
| with: | |
| access_token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/checkout@v3 | |
| - uses: coursier/setup-action@v1 | |
| with: | |
| jvm: temurin:1.11.0.17 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: build-target | |
| - name: Untar artifacts | |
| shell: bash | |
| run: tar xfz target.tgz | |
| - name: Cache ivy packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.ivy2/cache | |
| ~/.cache/coursier | |
| ~/.sbt | |
| key: sbt-cache-${{ hashFiles('**/*.sbt') }} | |
| restore-keys: sbt | |
| - name: Backend tests | |
| shell: bash | |
| run: ./ciRunSbt.sh test | |
| - name: Test Report | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: BackendTests-results-${{ matrix.scalaVersion }} | |
| path: '**/test-reports/*.xml' | |
| integrationTests: | |
| name: IntegrationTests | |
| needs: [ build, setup ] | |
| strategy: | |
| matrix: | |
| scalaVersion: ${{fromJson(needs.setup.outputs.scala_version_matrix)}} | |
| env: | |
| NUSSKNACKER_SCALA_VERSION: ${{ matrix.scalaVersion }} | |
| if: ${{ needs.setup.outputs.fe_changes_count != needs.setup.outputs.all_changes_count || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/demo' || startsWith(github.ref, 'refs/heads/preview') || startsWith(github.ref, 'refs/heads/release') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Cancel previous runs | |
| if: github.event_name != 'push' | |
| uses: styfle/[email protected] | |
| with: | |
| access_token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/checkout@v3 | |
| - uses: coursier/setup-action@v1 | |
| with: | |
| jvm: temurin:1.11.0.17 | |
| - name: Cache ivy packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.ivy2/cache | |
| ~/.cache/coursier | |
| ~/.sbt | |
| key: sbt-cache-${{ hashFiles('**/*.sbt') }} | |
| restore-keys: sbt | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: build-target | |
| - name: Untar artifacts | |
| shell: bash | |
| run: tar xfz target.tgz | |
| - name: Integration tests | |
| shell: bash | |
| env: | |
| dockerUpLatest: true | |
| run: ./ciRunSbt.sh It/test | |
| - name: Test Report | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: IntegrationTests-results-${{ matrix.scalaVersion }} | |
| path: '**/it-reports/*.xml' | |
| slowTests: | |
| name: SlowTests | |
| runs-on: ubuntu-latest | |
| needs: [ build, setup ] | |
| strategy: | |
| matrix: | |
| scalaVersion: ${{fromJson(needs.setup.outputs.scala_version_matrix)}} | |
| env: | |
| NUSSKNACKER_SCALA_VERSION: ${{ matrix.scalaVersion }} | |
| if: ${{ needs.setup.outputs.fe_changes_count != needs.setup.outputs.all_changes_count || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/demo' || startsWith(github.ref, 'refs/heads/preview') || startsWith(github.ref, 'refs/heads/release') }} | |
| steps: | |
| - name: Cancel previous runs | |
| if: github.event_name != 'push' | |
| uses: styfle/[email protected] | |
| with: | |
| access_token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/checkout@v3 | |
| - uses: coursier/setup-action@v1 | |
| with: | |
| jvm: temurin:1.11.0.17 | |
| - name: Cache ivy packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.ivy2/cache | |
| ~/.cache/coursier | |
| ~/.sbt | |
| key: sbt-cache-${{ hashFiles('**/*.sbt') }} | |
| restore-keys: sbt | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: build-target | |
| - name: Untar artifacts | |
| shell: bash | |
| run: tar xfz target.tgz | |
| # We run k8s tests here instead of in integrationTests job because it causes RAM overhead and some other docker tests failing occasionally | |
| - uses: AbsaOSS/[email protected] | |
| env: | |
| PROJECT_ROOT: ${{ github.workspace }} | |
| with: | |
| cluster-name: "k3s-default" | |
| k3d-version: "v5.6.0" | |
| args: >- | |
| --config=.k3d/single-cluster.yml | |
| --trace | |
| - name: Slow tests | |
| env: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_EVENT_HUBS_SHARED_ACCESS_KEY_NAME: ${{ secrets.AZURE_EVENT_HUBS_SHARED_ACCESS_KEY_NAME }} | |
| AZURE_EVENT_HUBS_SHARED_ACCESS_KEY: ${{ secrets.AZURE_EVENT_HUBS_SHARED_ACCESS_KEY }} | |
| shell: bash | |
| run: ./ciRunSbt.sh designer/Slow/test liteK8sDeploymentManager/ExternalDepsTests/test schemedKafkaComponentsUtils/ExternalDepsTests/test liteKafkaComponentsTests/ExternalDepsTests/test | |
| - name: docker logs | |
| if: success() || failure() | |
| run: | | |
| docker logs k3d-k3s-default-server-0 | |
| docker logs k3d-k3s-default-agent-0 | |
| docker logs k3d-k3s-default-serverlb | |
| - name: Test Report | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: SlowTests-results-${{ matrix.scalaVersion }} | |
| path: '**/test-reports/*.xml' | |
| frontendTests: | |
| name: FrontendTests | |
| runs-on: ubuntu-latest | |
| needs: [ setup ] | |
| env: | |
| # We can't just use conditional jobs mechanism ('if' directive) because 'publish' job depends on this one. | |
| shouldPerformFrontendTests: ${{ needs.setup.outputs.fe_changes_count > 0 }} | |
| steps: | |
| - name: Cancel previous runs | |
| if: ${{ env.shouldPerformFrontendTests == 'true' && github.event_name != 'push' }} | |
| uses: styfle/[email protected] | |
| with: | |
| access_token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/checkout@v3 | |
| if: ${{ env.shouldPerformFrontendTests == 'true' }} | |
| - name: Cache npm | |
| id: cache-npm | |
| if: ${{ env.shouldPerformFrontendTests == 'true' }} | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| **/node_modules | |
| ~/.cache/Cypress | |
| key: ${{ hashFiles('designer/client/package-lock.json', 'designer/client/.nvmrc', 'designer/submodules/package-lock.json', 'designer/submodules/.nvmrc') }} | |
| - name: Get Node.js version | |
| if: ${{ env.shouldPerformFrontendTests == 'true' }} | |
| id: nvm | |
| run: echo "NODE_VERSION=$(cat designer/client/.nvmrc)" >> $GITHUB_OUTPUT | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| if: ${{ env.shouldPerformFrontendTests == 'true' }} | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: ${{ steps.nvm.outputs.NODE_VERSION }} | |
| - name: Download node modules | |
| if: ${{ env.shouldPerformFrontendTests == 'true' && steps.cache-npm.outputs.cache-hit != 'true' }} | |
| run: | | |
| npm ci --no-audit --prefix designer/client | |
| npm ci --no-audit --prefix designer/submodules | |
| - name: Test FE | |
| if: ${{ env.shouldPerformFrontendTests == 'true' }} | |
| run: (cd designer/client; npm run test:unit) | |
| - name: Test Report | |
| if: ${{ env.shouldPerformFrontendTests == 'true' && (success() || failure()) }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: FrontendTests-results | |
| path: designer/client/junit.xml | |
| cypressTests: | |
| name: CypressTests | |
| runs-on: self-hosted | |
| needs: [ build, build-fe, setup ] | |
| env: | |
| # We skip docker build for fe-only changes. Would be more clean to split this step into two steps: build image and run tests | |
| # e.g. by using ishworkh/docker-image-artifact-upload/download but it caused ~3min overhead for the whole pipeline so we | |
| # have this conditional logic in this step. We force building images on our "special" branches because run between merges | |
| # could cause that cypress tests will be run at stale image (because of cancel-workflow-action). | |
| shouldBuildImage: ${{ needs.setup.outputs.fe_changes_count != needs.setup.outputs.all_changes_count || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/demo' || startsWith(github.ref, 'refs/heads/preview') || startsWith(github.ref, 'refs/heads/release') }} | |
| GIT_SOURCE_BRANCH: ${{ needs.setup.outputs.git_source_branch }} | |
| BE_PORT: 7251 | |
| steps: | |
| - name: Cancel previous runs | |
| if: github.event_name != 'push' | |
| uses: styfle/[email protected] | |
| with: | |
| access_token: ${{ secrets.GITHUB_TOKEN }} | |
| # On self-hosted runners, docker can have some obsolete containers. Because of that, we clean them all | |
| - name: Clean all docker containers | |
| shell: bash | |
| run: | | |
| c=$(docker ps -q) | |
| [[ $c ]] && docker kill $c || echo "No container run" | |
| - uses: actions/checkout@v3 | |
| - name: Get Node.js version | |
| id: nvm | |
| run: echo "NODE_VERSION=$(cat designer/client/.nvmrc)" >> $GITHUB_OUTPUT | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: ${{ steps.nvm.outputs.NODE_VERSION }} | |
| - uses: coursier/setup-action@v1 | |
| if: ${{ env.shouldBuildImage == 'true' }} | |
| with: | |
| jvm: temurin:1.11.0.17 | |
| - name: Download node modules | |
| run: | | |
| export npm_config_cache=$(mktemp -d) | |
| npm config ls -l | grep cache | |
| npm ci --no-audit --prefix designer/client | |
| npm ci --no-audit --prefix designer/submodules | |
| rm -rf $npm_config_cache | |
| - uses: actions/download-artifact@v4 | |
| if: ${{ env.shouldBuildImage == 'true' }} | |
| with: | |
| name: build-target | |
| - name: Untar artifacts | |
| if: ${{ env.shouldBuildImage == 'true' }} | |
| shell: bash | |
| run: tar xfz target.tgz | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: build-fe-dist | |
| - name: Untar fe artifacts | |
| shell: bash | |
| run: tar xfz fe-dist.tgz | |
| - name: set version | |
| if: ${{ env.shouldBuildImage == 'true' }} | |
| shell: bash | |
| run: echo "NUSSKNACKER_VERSION=${{ needs.setup.outputs.nk_snapshot_version }}" >> $GITHUB_ENV | |
| - name: Prepare docker | |
| if: ${{ env.shouldBuildImage == 'true' }} | |
| env: | |
| addDevArtifacts: true | |
| shell: bash | |
| #Doc generation is rather costly, we don't want it in test image creation | |
| run: sbt "set ThisBuild / version := \"$NUSSKNACKER_VERSION\"; set ThisBuild / packageDoc / publishArtifact := false; set Compile / doc / sources := Seq.empty" dist/Docker/publishLocal | |
| - name: FE tests e2e on build docker image | |
| if: ${{ env.shouldBuildImage == 'true' }} | |
| env: | |
| CYPRESS_SNAPSHOT_UPDATE: "true" | |
| CYPRESS_BASE_URL: http://localhost:${{ env.BE_PORT }} | |
| DOCKER_PULL_OPTION: never | |
| shell: bash | |
| run: | | |
| cd designer/client | |
| ! npx wait-on -t 250 tcp:localhost:${{ env.BE_PORT }} 2> /dev/null || (echo "Port: ${BE_PORT} already in use!" && exit 1) | |
| npx start-server-and-test backend:docker ${{ env.BE_PORT }} test:e2e | |
| - name: Determine docker tag version to use | |
| if: ${{ env.shouldBuildImage == 'false' }} | |
| shell: bash | |
| run: | | |
| # Take a look at build.sbt commonDockerSettings to see how this tag is determined. Thanks to fact that we publish all changes pushed to our "special" branches it should work quite correctly. | |
| NK_REF_VERSION=`[ "${GITHUB_REF}" != "" ] && echo "${GITHUB_REF}" | sed -e 's/refs\/heads\///g' -e 's/[^a-zA-Z0-9._-]/\_/g' -e 's/$/-latest/' | xargs -I VER sh -c 'docker pull touk/nussknacker:VER > /dev/null && echo VER || echo ""'` | |
| NK_BASE_REF_VERSION=`[ "${NK_REF_VERSION}" != "" ] && echo "${NK_REF_VERSION}" || [ "${GITHUB_BASE_REF}" != "" ] && echo "${GITHUB_BASE_REF}" | sed -e 's/refs\/heads\///g' -e 's/[^a-zA-Z0-9._-]/\_/g' -e 's/$/-latest/' | xargs -I VER sh -c 'docker pull touk/nussknacker:VER > /dev/null && echo VER || echo ""'` | |
| echo "NUSSKNACKER_VERSION=`[ \"${NK_BASE_REF_VERSION}\" != \"\" ] && echo \"${NK_BASE_REF_VERSION}\" || echo staging-latest`" >> $GITHUB_ENV | |
| - name: FE tests e2e on pulled image | |
| if: ${{ env.shouldBuildImage == 'false' }} | |
| env: | |
| CYPRESS_SNAPSHOT_UPDATE: "true" | |
| shell: bash | |
| run: | | |
| cd designer/client | |
| ! npx wait-on -t 250 tcp:localhost:${{ env.BE_PORT }} 2> /dev/null || (echo "Port: ${BE_PORT} already in use!" && exit 1) | |
| npx start-server-and-test backend:docker ${{ env.BE_PORT }} start-prod 3000 test:e2e | |
| - name: Test Report | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: CypressTests-results | |
| path: designer/client/cypress-test-results/*.xml | |
| - name: Store test results | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-test-results | |
| path: | | |
| docs/autoScreenshotChangeDocs/ | |
| designer/client/cypress/**/__image_snapshots__/ | |
| designer/client/cypress/screenshots/ | |
| designer/client/cypress/videos/ | |
| if-no-files-found: ignore | |
| - name: Create Pull Request | |
| id: update_snapshots | |
| uses: peter-evans/create-pull-request@v7 | |
| if: always() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) | |
| env: | |
| HASH: ${{ format('#{0}', github.event.number) }} | |
| BRANCH: ${{ needs.setup.outputs.git_source_branch }} | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| base: ${{ env.BRANCH }} | |
| branch: snapshots-patch/${{ env.BRANCH }} | |
| title: Update Cypress snapshots in ${{ env.BRANCH }} | |
| commit-message: Updated snapshots | |
| body: Updated snapshots in ${{ github.event_name == 'pull_request' && env.HASH || env.BRANCH}} | |
| labels: cypress | |
| - name: Comment PR | |
| if: ${{ always() && github.event_name == 'pull_request' && steps.update_snapshots.outputs.pull-request-number }} | |
| uses: thollander/actions-comment-pull-request@v2 | |
| with: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| message: | | |
| ${{ steps.update_snapshots.outputs.pull-request-operation }}: #${{ steps.update_snapshots.outputs.pull-request-number }} | |
| :warning: Be careful! Snapshot changes are not necessarily the cause of the error. Check the logs. | |
| comment_tag: snapshots_pr | |
| - name: Force fail if update PR created | |
| if: ${{ always() && github.event_name == 'pull_request' && steps.update_snapshots.outputs.pull-request-number }} | |
| uses: actions/[email protected] | |
| with: | |
| script: | | |
| core.setFailed('Snapshots updated, chceck comments!') | |
| #TODO: extract to different workflow? | |
| publish: | |
| runs-on: ubuntu-latest | |
| needs: [build, build-fe, setup, tests, integrationTests, slowTests, frontendTests, cypressTests] | |
| #TODO: should we publish everything on all those branches? | |
| if: ${{ github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/demo' || startsWith(github.ref, 'refs/heads/preview') || startsWith(github.ref, 'refs/heads/release') }} | |
| env: | |
| #TODO: shouldn't be needed... | |
| nexusUrl: https://oss.sonatype.org/content/repositories/snapshots | |
| # We add dev model to staging because it is used in dev used on this environment, but we also add it for other branches for purpose of cypress tests | |
| addDevArtifacts: true | |
| NUSSKNACKER_VERSION: ${{ needs.setup.outputs.nk_snapshot_version }} | |
| GIT_SOURCE_BRANCH: ${{ needs.setup.outputs.git_source_branch }} | |
| SONATYPE_USERNAME: ${{ secrets.SONATYPE_USER }} | |
| SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} | |
| steps: | |
| - name: Cancel previous runs | |
| if: github.event_name != 'push' | |
| uses: styfle/[email protected] | |
| with: | |
| access_token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/checkout@v3 | |
| - uses: coursier/setup-action@v1 | |
| with: | |
| jvm: temurin:1.11.0.17 | |
| - name: Cache ivy packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.ivy2/cache | |
| ~/.cache/coursier | |
| ~/.sbt | |
| key: sbt-cache-${{ hashFiles('**/*.sbt') }} | |
| restore-keys: sbt | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: build-target | |
| - name: Untar artifacts | |
| shell: bash | |
| run: tar xfz target.tgz | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: build-fe-dist | |
| - name: Untar fe artifacts | |
| shell: bash | |
| run: tar xfz fe-dist.tgz | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USER }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Setup buildx builder | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| platforms: linux/amd64,linux/arm64 | |
| config-inline: | | |
| [worker.oci] | |
| max-parallelism = 1 | |
| - name: Try to free space | |
| run: | | |
| # Workaround to prevent "no space left on device" during buildx build - https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| sudo rm -rf /usr/local/lib/android | |
| - name: Publish with scaladocs | |
| if: ${{ github.ref == 'refs/heads/staging' }} | |
| env: | |
| # Workaround to preveent "no space left on device" during publishDocker failed to create temp dir: mkdir /tmp/containerd-mountXYZ | |
| TMPDIR: ${{ github.workspace }} | |
| shell: bash | |
| #TODO: handle version better, do we want to publish docker image for older scala versions? If so, how should it be tagged? | |
| run: sbt "set ThisBuild / version := \"$NUSSKNACKER_VERSION\"" +publish +dist/Docker/publish +liteEngineRuntimeApp/Docker/publish | |
| - name: Publish without scaladocs | |
| if: ${{ github.ref != 'refs/heads/staging' }} | |
| env: | |
| # Workaround to preveent "no space left on device" during publishDocker failed to create temp dir: mkdir /tmp/containerd-mountXYZ | |
| TMPDIR: ${{ github.workspace }} | |
| shell: bash | |
| #TODO: handle version better, do we want to publish docker image for older scala versions? If so, how should it be tagged? | |
| run: sbt "set ThisBuild / version := \"$NUSSKNACKER_VERSION\"; set ThisBuild / packageDoc / publishArtifact := false; set Compile / doc / sources := Seq.empty" +publish +dist/Docker/publish +liteEngineRuntimeApp/Docker/publish | |
| trigger-external-build: | |
| runs-on: ubuntu-latest | |
| needs: [ setup, publish ] | |
| if: ${{ github.ref == 'refs/heads/staging' }} | |
| strategy: | |
| matrix: | |
| include: | |
| - repo: nussknacker-helm | |
| workflow_id: helm-test-workflow.yaml | |
| ref: main | |
| - repo: nussknacker-quickstart | |
| workflow_id: pr.yml | |
| ref: staging | |
| - repo: nussknacker-sample-components | |
| workflow_id: pr.yml | |
| ref: staging | |
| - repo: nussknacker-flink-compatibility | |
| workflow_id: test.yml | |
| ref: staging | |
| steps: | |
| - uses: actions/github-script@v6 | |
| with: | |
| github-token: ${{ secrets.ORG_TRIGGER_PAT_TOKEN }} | |
| script: | | |
| await github.rest.actions.createWorkflowDispatch({ | |
| owner: context.repo.owner, | |
| repo: '${{ matrix.repo }}', | |
| workflow_id: '${{ matrix.workflow_id }}', | |
| ref: '${{ matrix.ref }}', | |
| inputs: {"nussknacker_version": "${{ needs.setup.outputs.nk_snapshot_version }}"} | |
| }) |