Update_Apps #1493
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update_Apps | |
| # Controls when the workflow will run | |
| on: | |
| # run at 00:00 and 12:00 UTC every Day | |
| schedule: | |
| - cron: "0 0,12 * * *" | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| # A workflow run is made up of one or more jobs that can run sequentially or in parallel | |
| # this workflow contains 3 jobs that run sequentially: get-latest-app-versions, test-updated-apps, and create-pr | |
| # get-latest-app-versions gets the latests app versions and modifies the working directory of the repository with those changes. then it uploads that working directory of the git repository as an artifact for use in later jobs | |
| # test-updated-apps runs a matrix of testcases where all updated apps from the first job are tested for successful installation/uninstallation in the working directory of the git repoistory downloaded from the artifact | |
| # create-pr then uses the output from get-latest-app-versions and test-updated-apps to finalize a PR where only apps that passed all testcases are updated and any that fail have a descriptive error written in the PR comment | |
| jobs: | |
| get-latest-app-versions: | |
| name: Get Latest App Versions | |
| # The type of runner that the job will run on | |
| runs-on: ubuntu-latest | |
| outputs: | |
| UPDATED_APPS: ${{ steps.updated.outputs.UPDATED_APPS }} | |
| FAILED_APPS: ${{ steps.failed.outputs.FAILED_APPS }} | |
| # Steps represent a sequence of tasks that will be executed as part of the job | |
| steps: | |
| # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it | |
| - uses: actions/checkout@v4 | |
| # Runs a set of commands using the runners shell | |
| - name: Run app update scripts | |
| env: | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| run: | | |
| # print user info | |
| echo $USER $USERNAME $(id) $(whoami) | |
| #source pi-apps functions | |
| #export all functions and variables | |
| set -a | |
| #make DIRECTORY equal to GITHUB_WORKSPACE, for subscripts and api functions | |
| DIRECTORY=$GITHUB_WORKSPACE | |
| source $GITHUB_WORKSPACE/api | |
| #add special functions | |
| get_release() { | |
| curl -s --header "Authorization: token $GH_PERSONAL_ACCESS_TOKEN" "https://api.github.com/repos/$1/releases/latest" | jq -r '.tag_name' | sed s/^v//g | |
| } | |
| get_release_raw() { | |
| curl -s --header "Authorization: token $GH_PERSONAL_ACCESS_TOKEN" "https://api.github.com/repos/$1/releases/latest" | jq -r '.tag_name' | |
| } | |
| get_prerelease() { | |
| curl -s --header "Authorization: token $GH_PERSONAL_ACCESS_TOKEN" "https://api.github.com/repos/$1/releases" | jq -r 'map(select(.prerelease)) | first | .tag_name' | sed s/^v//g | |
| } | |
| get_prerelease_raw() { | |
| curl -s --header "Authorization: token $GH_PERSONAL_ACCESS_TOKEN" "https://api.github.com/repos/$1/releases" | jq -r 'map(select(.prerelease)) | first | .tag_name' | |
| } | |
| function validate_url(){ | |
| if command wget --timeout=5 -q --spider "$1"; then | |
| return 0 | |
| else | |
| return 1 | |
| fi | |
| } | |
| #stop exporting functions | |
| set +a | |
| #make sure all update scripts are executable | |
| chmod +x $GITHUB_WORKSPACE/.github/workflows/updates/*.sh | |
| cd $GITHUB_WORKSPACE | |
| apps=( .github/workflows/updates/*.sh ) | |
| for app_directory in "${apps[@]}"; do | |
| echo | |
| #make sure we are still in the main workspace (incase an update script left off elsewhere) | |
| cd $GITHUB_WORKSPACE | |
| export app_name="$(echo ${app_directory%.*} | sed 's:.*/::')" | |
| echo "$app_name" | |
| status "Checking $app_name for updates" | |
| # move to app folder | |
| cd "$GITHUB_WORKSPACE/apps/$app_name" | |
| # run app update script | |
| "$GITHUB_WORKSPACE/$app_directory" | |
| done | |
| cd | |
| - name: Output updated apps | |
| id: updated | |
| run: | | |
| if test -f /tmp/updated_apps; then | |
| sort -u /tmp/updated_apps > /tmp/updated_apps_sorted | |
| echo "UPDATED_APPS<<EOF" >> $GITHUB_OUTPUT | |
| cat /tmp/updated_apps_sorted >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Output failed apps | |
| id: failed | |
| run: | | |
| if test -f /tmp/failed_apps; then | |
| echo "FAILED_APPS<<EOF" >> $GITHUB_OUTPUT | |
| cat /tmp/failed_apps | sed '0~1 a\\' >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Tar GITHUB_WORKSPACE | |
| run: | | |
| cd ${{ runner.temp }} | |
| tar -C $GITHUB_WORKSPACE -cvf GITHUB_WORKSPACE.tar . | |
| - name: Upload GITHUB_WORKSPACE Artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: GITHUB_WORKSPACE | |
| path: ${{ runner.temp }}/GITHUB_WORKSPACE.tar | |
| test-updated-apps: | |
| runs-on: ubuntu-latest | |
| needs: get-latest-app-versions | |
| strategy: | |
| matrix: | |
| include: | |
| - os: bullseye | |
| arch: armhf | |
| image: https://downloads.raspberrypi.org/raspios_oldstable_armhf/images/raspios_oldstable_armhf-2023-12-06/2023-12-05-raspios-bullseye-armhf.img.xz | |
| cache: yes | |
| - os: bullseye | |
| arch: arm64 | |
| image: https://downloads.raspberrypi.org/raspios_oldstable_arm64/images/raspios_oldstable_arm64-2023-12-06/2023-12-05-raspios-bullseye-arm64.img.xz | |
| cache: yes | |
| - os: bookworm | |
| arch: armhf | |
| image: https://downloads.raspberrypi.com/raspios_armhf/images/raspios_armhf-2024-07-04/2024-07-04-raspios-bookworm-armhf.img.xz | |
| cache: yes | |
| - os: bookworm | |
| arch: arm64 | |
| image: https://downloads.raspberrypi.com/raspios_arm64/images/raspios_arm64-2024-07-04/2024-07-04-raspios-bookworm-arm64.img.xz | |
| cache: yes | |
| name: Testing on ${{ matrix.os }} - ${{ matrix.arch }} | |
| outputs: | |
| FAILED_UPDATE_APPS_bullseye_armhf: ${{ steps.failed.outputs.FAILED_UPDATE_APPS_bullseye_armhf || '' }} | |
| FAILED_UPDATE_APPS_bullseye_arm64: ${{ steps.failed.outputs.FAILED_UPDATE_APPS_bullseye_arm64 || '' }} | |
| FAILED_UPDATE_APPS_bookworm_armhf: ${{ steps.failed.outputs.FAILED_UPDATE_APPS_bookworm_armhf || '' }} | |
| FAILED_UPDATE_APPS_bookworm_arm64: ${{ steps.failed.outputs.FAILED_UPDATE_APPS_bookworm_arm64 || '' }} | |
| UPDATED_APPS_bullseye_armhf: ${{ steps.updated.outputs.UPDATED_APPS_bullseye_armhf || '' }} | |
| UPDATED_APPS_bullseye_arm64: ${{ steps.updated.outputs.UPDATED_APPS_bullseye_arm64 || '' }} | |
| UPDATED_APPS_bookworm_armhf: ${{ steps.updated.outputs.UPDATED_APPS_bookworm_armhf || '' }} | |
| UPDATED_APPS_bookworm_arm64: ${{ steps.updated.outputs.UPDATED_APPS_bookworm_arm64 || '' }} | |
| steps: | |
| # restore GITHUB_WORKSPACE | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: GITHUB_WORKSPACE | |
| path: ${{ runner.temp }} | |
| - name: Extract GITHUB_WORKSPACE | |
| run: | | |
| cd ${{ runner.temp }} | |
| tar xf GITHUB_WORKSPACE.tar -C $GITHUB_WORKSPACE | |
| cd $GITHUB_WORKSPACE | |
| - name: Replace secret reference with contents | |
| if: ${{ matrix.image == 'RPIOS_TRIXIE_URL' || matrix.image == 'RPIOS_TRIXIE_ARMHF_URL' }} | |
| run: | | |
| image_url=${{ secrets[matrix.image] }} | |
| echo "::add-mask::$image_url" | |
| echo "image_url=$image_url" >> "$GITHUB_ENV" | |
| - name: Replace non-secret reference with contents | |
| if: ${{ matrix.image != 'RPIOS_TRIXIE_URL' && matrix.image != 'RPIOS_TRIXIE_ARMHF_URL' }} | |
| run: | | |
| image_url=${{ matrix.image }} | |
| echo "image_url=$image_url" >> "$GITHUB_ENV" | |
| - name: Write UPDATED_APPS and FAILED_APPS from OUTPUT of get-latest-app-versions to ENV | |
| run: | | |
| echo "UPDATED_APPS<<EOF" >> $GITHUB_ENV | |
| echo "${{needs.get-latest-app-versions.outputs.UPDATED_APPS}}" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| echo "FAILED_APPS<<EOF" >> $GITHUB_ENV | |
| echo "${{needs.get-latest-app-versions.outputs.FAILED_APPS}}" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Test installing updated apps on ${{ matrix.os }} - ${{ matrix.arch }} | |
| uses: theofficialgman/arm-runner-action@v12 | |
| with: | |
| base_image: ${{ env.image_url }} | |
| # bind mount the directory so any changes propogate to outside the chroot | |
| bind_mount_repository: yes | |
| # give the image more space | |
| image_additional_mb: 5000 | |
| # disable image caching to not leak the URL on some systems | |
| enable_image_caching: ${{ matrix.cache }} | |
| # set CPUs to use | |
| cpu: cortex-a7:cortex-a72 | |
| # use custom /proc/cpuinfo | |
| cpu_info: cpuinfo/raspberrypi_4b | |
| # user runner name as default path | |
| copy_repository_path: /home/runner/pi-apps | |
| # export github env back to outside the chroot | |
| export_github_env: yes | |
| import_github_env: true | |
| # disable debug which shows all commands to be executed | |
| debug: no | |
| # set shell to bash | |
| shell: /bin/bash | |
| commands: | | |
| sudo chown runner:docker /home/runner | |
| # print user info | |
| echo $USER $USERNAME $(id) $(whoami) | |
| sudo bash -c 'echo $USER $USERNAME $(id) $(whoami)' | |
| # create standard directories | |
| mkdir -p $HOME/.local/share/applications $HOME/.local/bin | |
| sudo mkdir -p /usr/local/bin /usr/local/share/applications | |
| # install pi-apps dependencies | |
| sudo apt update | |
| sudo apt install -y yad curl wget aria2 lsb-release software-properties-common apt-utils imagemagick bc librsvg2-bin locales shellcheck git wmctrl xdotool x11-utils rsync | |
| # runonce-entries is run in the build tester, runonce requires that all api functions be available to subprocess (like is done in the gui script) | |
| #for the will_reinstall() and list_intersect() functions | |
| set -a #make all functions in the api available to subprocesses | |
| #make DIRECTORY equal to GITHUB_WORKSPACE, for subscripts and api functions | |
| DIRECTORY=$(pwd) | |
| source "${DIRECTORY}/api" || error "failed to source ${DIRECTORY}/api" | |
| #Run runonce entries | |
| "${DIRECTORY}/etc/runonce-entries" | |
| set +a #stop exporting functions | |
| cd ${DIRECTORY} | |
| # upgrade cmake to 3.20+ from theofficialgman ppa to fix QEMU only issue https://gitlab.kitware.com/cmake/cmake/-/issues/20568 | |
| package_is_new_enough cmake 3.20 || debian_ppa_installer "theofficialgman/cmake-bionic" "bionic" "0ACACB5D1E74E484" || exit 1 | |
| # store apps changed from last commit to working directory in variable | |
| mapfile -t changed_apps < <(git diff --name-only | grep ^apps/ | awk -F '/' '{print $2}' | sort -u) | |
| # clean out any app status files | |
| rm -rf ./data/status | |
| # attempt to install updated apps using manage script loop | |
| # if any app fails, add it to the FAILED_INSTALL_APPS or FAILED_UNINSTALL_APPS variable | |
| if [[ ${{ matrix.arch }} == armhf ]]; then | |
| # skip checking DDNet (due to Rust) if on arm32 under QEMU: https://github.com/rust-lang/cargo/issues/8719 | |
| # skip checking Microsoft PowerShell due to QEMU instability with .NET: https://github.com/Botspot/pi-apps/pull/2252#issuecomment-1403043945 | |
| # skip WPS Office on 32bit as it requires "64bit" kernel to be detected. QEMU reports an armv7 kernel. WPS Office is still tested on arm64 (which will verify if the deb works and installs correctly) | |
| # skip Boxy SVG on 32bit due to QEMU instability | |
| skiplist="DDNet | |
| Microsoft PowerShell | |
| WPS Office | |
| Boxy SVG" | |
| else | |
| # skip checking Microsoft PowerShell due to QEMU instability with .NET: https://github.com/Botspot/pi-apps/pull/2252#issuecomment-1403043945 | |
| skiplist="Microsoft PowerShell" | |
| fi | |
| for app in "${changed_apps[@]}"; do | |
| # only attempt install/uninstall if updated app is supported on this architecture | |
| scriptname="$(script_name_cpu "$app")" #will be install, install-32, install-64, or package | |
| if [ -z "$scriptname" ];then | |
| continue | |
| fi | |
| # skip app if is in the skiplist | |
| if echo "$skiplist" | grep -Fxq "$app"; then | |
| continue | |
| fi | |
| ./manage install "$app" || { FAILED_INSTALL_APPS+="$app"$'\n'; UPDATED_APPS=$(echo "$UPDATED_APPS" | sed "/$app/d"); } | |
| ./manage uninstall "$app" || { FAILED_UNINSTALL_APPS+="$app"$'\n'; UPDATED_APPS=$(echo "$UPDATED_APPS" | sed "/$app/d"); } | |
| done | |
| if [ ! -z "$FAILED_INSTALL_APPS" ] || [ ! -z "$FAILED_UNINSTALL_APPS" ]; then | |
| echo "FAILED_UPDATE_APPS<<EOF" >> $GITHUB_ENV | |
| echo "$FAILED_INSTALL_APPS$FAILED_UNINSTALL_APPS" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| echo "UPDATED_APPS<<EOF" >> $GITHUB_ENV | |
| echo "$UPDATED_APPS" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| fi | |
| - name: Write FAILED_APPS to OUTPUT | |
| id: failed | |
| run: | | |
| echo "FAILED_UPDATE_APPS_${{ matrix.os }}_${{ matrix.arch }}<<EOF" >> $GITHUB_OUTPUT | |
| echo "${{env.FAILED_UPDATE_APPS}}" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| - name: Write UPDATED_APPS to OUTPUT | |
| id: updated | |
| run: | | |
| echo "UPDATED_APPS_${{ matrix.os }}_${{ matrix.arch }}<<EOF" >> $GITHUB_OUTPUT | |
| echo "${{env.UPDATED_APPS}}" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| create-commit: | |
| needs: [get-latest-app-versions,test-updated-apps] | |
| runs-on: ubuntu-latest | |
| steps: | |
| # setup github environment variables | |
| - uses: actions/checkout@v4 | |
| # restore GITHUB_WORKSPACE | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: GITHUB_WORKSPACE | |
| path: ${{ runner.temp }} | |
| - name: Extract GITHUB_WORKSPACE | |
| run: | | |
| cd ${{ runner.temp }} | |
| tar xf GITHUB_WORKSPACE.tar -C $GITHUB_WORKSPACE --exclude=".git" | |
| cd $GITHUB_WORKSPACE | |
| - name: Delete artifact | |
| uses: geekyeggo/delete-artifact@v5 | |
| with: | |
| name: GITHUB_WORKSPACE | |
| failOnError: false | |
| - name: Revert failed apps and generate list | |
| run: | | |
| ALL_FAILED_APPS="${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bullseye_armhf}}"$'\n'"${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bullseye_arm64}}"$'\n'"${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bookworm_armhf}}"$'\n'"${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bookworm_arm64}}" | |
| ALL_FAILED_APPS="$(echo "$ALL_FAILED_APPS" | sort -u | awk NF)" | |
| echo "ALL_FAILED_APPS<<EOF" >> $GITHUB_ENV | |
| echo "$ALL_FAILED_APPS" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| error_string="" | |
| IFS=$'\n' | |
| for app in $ALL_FAILED_APPS; do | |
| git checkout -- "apps/$app" | |
| error_string+='![badge-error][badge-error]'" Failed to install $app on " | |
| if echo "${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bullseye_armhf}}" | grep -Fxq "$app"; then | |
| error_string+="bullseye armhf, " | |
| fi | |
| if echo "${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bullseye_arm64}}" | grep -Fxq "$app"; then | |
| error_string+="bullseye arm64, " | |
| fi | |
| if echo "${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bookworm_armhf}}" | grep -Fxq "$app"; then | |
| error_string+="bookworm armhf, " | |
| fi | |
| if echo "${{needs.test-updated-apps.outputs.FAILED_UPDATE_APPS_bookworm_arm64}}" | grep -Fxq "$app"; then | |
| error_string+="bookworm arm64, " | |
| fi | |
| error_string+="reverting to previous version."$'\n'$'\n' | |
| done | |
| true | |
| echo "ALL_FAILED_APPS_ERROR_STRING<<EOF" >> $GITHUB_ENV | |
| echo "$error_string" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Generate updated apps list | |
| run: | | |
| list_subtract() { #Outputs a list of apps from stdin, minus the ones that appear in $1 using the UPDATED_APP format | |
| # for example, the following two inputs will be a match | |
| # - BlockBench-arm64: 4.7.4 -> 4.8.0 | |
| # BlockBench | |
| # change \n to -\n | change '^' to '- ' | |
| grep -ve "$(echo "$1" | sed -z 's/\n/\-\n/g' | sed 's/^/- /g')" | |
| true | |
| } | |
| ALL_UPDATED_APPS="${{needs.test-updated-apps.outputs.UPDATED_APPS_bullseye_armhf}}"$'\n'"${{needs.test-updated-apps.outputs.UPDATED_APPS_bullseye_arm64}}"$'\n'"${{needs.test-updated-apps.outputs.UPDATED_APPS_bookworm_armhf}}"$'\n'"${{needs.test-updated-apps.outputs.UPDATED_APPS_bookworm_arm64}}" | |
| ALL_UPDATED_APPS="$(echo "$ALL_UPDATED_APPS" | sort -u | awk NF)" | |
| [ ! -z "$ALL_UPDATED_APPS" ] && ALL_UPDATED_APPS="$(echo "$ALL_UPDATED_APPS" | list_subtract "$ALL_FAILED_APPS")" | |
| [ ! -z "$ALL_UPDATED_APPS" ] && COMMIT_TITLE="$(echo "$ALL_UPDATED_APPS" | awk -F '^- |-all:|-armhf:|-arm64:' '{print $2}' | sort -u | sed ':a;N;$!ba;s/\n/, /g' | sed -e 's/$/: Update App Versions/')" | |
| echo "ALL_UPDATED_APPS<<EOF" >> $GITHUB_ENV | |
| echo "$ALL_UPDATED_APPS" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| echo "COMMIT_TITLE<<EOF" >> $GITHUB_ENV | |
| echo "$COMMIT_TITLE" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Create Commit and Push | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| commit_author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> | |
| commit_message: | | |
| ${{ env.COMMIT_TITLE }} | |
| ${{ env.ALL_UPDATED_APPS }} | |
| - name: Create or Update issue | |
| # don't error if this step fails | |
| # an upstream bug with no fix causes this to sometimes fail https://github.com/JasonEtco/create-an-issue/issues/142 | |
| continue-on-error: true | |
| if: ${{ needs.get-latest-app-versions.outputs.FAILED_APPS != '' || env.ALL_FAILED_APPS_ERROR_STRING != '' }} | |
| uses: JasonEtco/create-an-issue@v2 | |
| env: | |
| FAILED_APPS: ${{needs.get-latest-app-versions.outputs.FAILED_APPS}} | |
| ALL_FAILED_APPS_ERROR_STRING: ${{ env.ALL_FAILED_APPS_ERROR_STRING }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_RUN_ID: ${{ github.run_id }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| update_existing: true | |
| search_existing: all | |
| filename: .github/update_apps_issue_template.md |