ci: update tests, CI & modpathfile

* workaround intermittent macos CI matplotlib failures (modflowpy#1491)
* don't use plt.show() in tests
* add explanatory comments to conftest.py
* skip ex-gwtgwt-mt3dms-p10 mf6 example test (MODFLOW-USGS/modflow6#1008)
* give test_mt3d.py::test_mfnwt_CrnkNic more retries
* rename release/ to scripts/
* move pull_request_prepare.py to scripts/
* separate CI workflows for benchmarks, examples and regression test
* name benchmark artifacts benchmarks-<system>-python version>-<workflow run ID>
* add postprocess_benchmarks.py to scripts/
* add benchmark postprocessing CI job (creates artifact benchmarks-<workflow run ID>)
* define actions to cache modflow exes (+ invalidate on new release)
* move sort to child classes' __init__() from _ModpathSeries.get_data() (address modflowpy#1479)
* reenable PathlineFile.get_destination_pathline_data() benchmark
* add test that PathlineFile sorts on initialization
* update ci.yml usages to commit.yml
* don't upload coverage after smoke tests, benchmarks, regression tests and example tests
* upload coverage on PR as well as push (fix codecov bot comments)
* update to codecov action v3
* decrease coverage precision to 2 places (avoid small deltas)

.github/actions/cache_exes/action.yml

@@ -0,0 +1,62 @@
+name: Cache Modflow executables
+description: 'Cache MODFLOW executables from the MODFLOW-USGS/executables repository'
+inputs:
+  path:
+    description: 'The path to store the executables (e.g. a bin directory)'
+    required: true
+    default: 'bin'
+  os:
+    description: 'The runner operating system'
+    required: true
+  github_token:
+    description: 'The GitHub API access token'
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Check release
+      shell: bash
+      run: |
+        # get info for the executables repository's latest release
+        release_json=$(gh api -X GET -H "Accept: application/vnd.github+json" /repos/MODFLOW-USGS/executables/releases/latest)
+
+        # get asset ID of the release's metadata file, if one exists
+        get_asset_id="
+        import json
+        import sys
+        release = json.load(sys.stdin, strict=False)
+        metadata = next(iter([a for a in release['assets'] if a['name'] == 'code.json']), None)
+        print(dict(metadata)['id'] if metadata else '')
+        "
+        asset_id=$(echo "$release_json" | python -c "$get_asset_id")
+
+        # asset_id is empty if metadata file asset wasn't found
+        if [ ${#asset_id} -gt 0 ]; then
+           gh api -H "Accept: application/octet-stream" "/repos/MODFLOW-USGS/executables/releases/assets/$asset_id" >> executables.json
+        else
+          # give hashFiles an empty file to hash
+          touch executables.json
+        fi
+      env:
+        GH_TOKEN: ${{ inputs.github_token }}
+
+    - name: Cache executables
+      id: cache_executables
+      uses: actions/cache@v3
+      with:
+        path: ${{ inputs.path }}
+        key: modflow-exes-${{ inputs.os }}-${{ hashFiles('executables.json') }}
+
+    - name: Install executables
+      shell: bash
+      working-directory: ./autotest
+      run: |
+        mkdir -p ${{ inputs.path }}
+
+        # if executables exist, get-modflow won't re-download without --force
+        get-modflow ${{ inputs.path }}
+
+        # add bin directory to runner's PATH
+        echo ${{ inputs.path }} >> $GITHUB_PATH
+      env:
+        GITHUB_TOKEN: ${{ inputs.github_token }}

.github/actions/cache_exes_win/action.yml

@@ -0,0 +1,56 @@
+name: 'Cache executables (Windows)'
+description: 'Cache MODFLOW executables from the MODFLOW-USGS/executables repository (Windows)'
+inputs:
+  path:
+    description: 'The path to store the executables (e.g. a bin directory)'
+    required: true
+    default: 'bin'
+  os:
+    description: 'The runner operating system'
+    required: true
+  github_token:
+    description: 'The GitHub API access token'
+    required: true
+runs:
+  using: "composite"
+  steps:
+  - name: Check release
+    shell: pwsh
+    run: |
+      # get info for the executables repository's latest release
+      $release_json=(gh api -X GET -H "Accept: application/vnd.github+json" /repos/MODFLOW-USGS/executables/releases/latest)
+
+      # get asset ID of the release's metadata file, if one exists
+      $pattern="code.json"
+      $release=(echo $release_json | ConvertFrom-Json)
+      $asset_id=($release.assets | Where-Object {$_.name -match "$pattern"} | % {echo $_.id})
+
+      # asset_id is empty if metadata file asset wasn't found
+      if ($asset_id.Length -gt 0) {
+        gh api -H "Accept: application/octet-stream" "/repos/MODFLOW-USGS/executables/releases/assets/$asset_id" >> executables.json
+      } else {
+        # give hashFiles an empty file to hash
+        New-Item -Name "executables.json" -ItemType File
+      }
+    env:
+      GH_TOKEN: ${{ inputs.github_token }}
+
+  - name: Cache executables
+    id: cache_executables
+    uses: actions/cache@v3
+    with:
+      path: ${{ inputs.path }}
+      key: modflow-exes-${{ inputs.os }}-${{ hashFiles('executables.json') }}
+
+  - name: Install executables
+    shell: pwsh
+    run: |
+      md -Force ${{ inputs.path }}
+
+      # if executables exist, get-modflow won't re-download without --force
+      get-modflow ${{ inputs.path }}
+
+      # add bin directory to runner's PATH
+      echo ${{ inputs.path }} | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+    env:
+      GITHUB_TOKEN: ${{ inputs.github_token }}

.github/workflows/benchmark.yml

@@ -0,0 +1,246 @@
+name: FloPy benchmarks
+
+on:
+  push:
+    branches:
+      - develop
+      - tests
+  pull_request:
+    branches:
+      - develop
+      - tests
+  schedule:
+    - cron: '0 8 * * *' # run at 8 AM UTC (12 am PST)
+
+jobs:
+  benchmark:
+    name: Benchmarks
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, macos-latest ]
+        python-version: [ 3.7, 3.8, 3.9, "3.10" ]
+        exclude:
+          # avoid shutil.copytree infinite recursion bug
+          # https://github.com/python/cpython/pull/17098
+          - python-version: '3.8.0'
+        include:
+          - os: ubuntu-latest
+            path: ~/.cache/pip
+          - os: macos-latest
+            path: ~/Library/Caches/pip
+    defaults:
+      run:
+        shell: bash
+    timeout-minutes: 90
+
+    steps:
+      - name: Checkout repo
+        uses: actions/[email protected]
+
+      - name: Cache Python
+        uses: actions/cache@v3
+        with:
+          path: ${{ matrix.path }}
+          key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-pip-
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Get branch name
+        uses: nelonoel/[email protected]
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+          pip install ".[test, optional]"
+
+      - name: Extract home dir
+        id: get_home_dir
+        run: |
+          echo "::set-output name=home_dir::$HOME"
+
+      - name: Install Modflow executables
+        uses: ./.github/actions/cache_exes
+        with:
+          path: ${{ steps.get_home_dir.outputs.home_dir }}/.local/bin
+          os: ${{ matrix.os }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run benchmarks
+        working-directory: ./autotest
+        run: |
+          mkdir -p .benchmarks
+          pytest -v --durations=0 --benchmark-only --benchmark-json .benchmarks/${{ matrix.os }}_python${{ matrix.python-version }}.json --keep-failed=.failed
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload failed benchmark artifact
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: failed-benchmark-${{ matrix.os }}-${{ matrix.python-version }}-${{ github.run_id }}
+          path: |
+            ./autotest/.failed/**
+
+      - name: Upload benchmark result artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: benchmarks-${{ matrix.os }}-${{ matrix.python-version }}-${{ github.run_id }}
+          path: |
+            ./autotest/.benchmarks/**/*.json
+
+  benchmark_windows:
+    name: Benchmarks (Windows)
+    runs-on: windows-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ 3.7, 3.8, 3.9, "3.10" ]
+        exclude:
+          # avoid shutil.copytree infinite recursion bug
+          # https://github.com/python/cpython/pull/17098
+          - python-version: '3.8.0'
+    defaults:
+      run:
+        shell: pwsh
+    timeout-minutes: 90
+
+    steps:
+      - name: Checkout repo
+        uses: actions/[email protected]
+
+      - name: Get branch name
+        uses: nelonoel/[email protected]
+
+      - name: Cache Miniconda
+        uses: actions/cache@v3
+        with:
+          path: ~/conda_pkgs_dir
+          key: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.run-type }}-${{ hashFiles('etc/environment.yml') }}
+
+      # Standard python fails on windows without GDAL installation
+      # Using custom bash shell ("shell: bash -l {0}") with Miniconda
+      - name: Setup Miniconda
+        uses: conda-incubator/[email protected]
+        with:
+          python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+          auto-update-conda: true
+          activate-environment: flopy
+          use-only-tar-bz2: true
+
+      - name: Install Python dependencies
+        run: |
+          conda env update --name flopy --file etc/environment.yml
+          python -m pip install --upgrade pip
+          pip install https://github.com/modflowpy/pymake/zipball/master
+          pip install xmipy
+          pip install .
+
+      - name: Install Modflow executables
+        uses: ./.github/actions/cache_exes_win
+        with:
+          path: C:\Users\runneradmin\.local\bin
+          os: ${{ runner.os }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run benchmarks
+        working-directory: ./autotest
+        run: |
+          md -Force .benchmarks
+          pytest -v --durations=0 --benchmark-only --benchmark-json .benchmarks/${{ runner.os }}_python${{ matrix.python-version }}.json --keep-failed=.failed
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload failed benchmark artifact
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: failed-benchmark-${{ runner.os }}-${{ matrix.python-version }}-${{ github.run_id }}
+          path: |
+            ./autotest/.failed/**
+
+      - name: Upload benchmark result artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: benchmarks-${{ runner.os }}-${{ matrix.python-version }}-${{ github.run_id }}
+          path: |
+            ./autotest/.benchmarks/**/*.json
+
+  post_benchmark:
+    needs:
+      - benchmark
+      - benchmark_windows
+    name: Process benchmark results
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout repo
+        uses: actions/[email protected]
+
+      - name: Cache Python
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-3.7-pip-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-3.7-pip-
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.7
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install numpy pandas matplotlib seaborn
+
+      - name: Download all artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: ./autotest/.benchmarks
+
+      - name: Process benchmark results
+        run: |
+          artifact_json=$(gh api -X GET -H "Accept: application/vnd.github+json" /repos/modflowpy/flopy/actions/artifacts)
+          get_artifact_ids="
+          import json
+          import sys
+          from os import linesep
+
+          artifacts = json.load(sys.stdin, strict=False)['artifacts']
+          artifacts = [a for a in artifacts if a['name'].startswith('benchmarks-') and a['name'].split('-')[-1].isdigit()]
+
+          print(linesep.join([str(a['id']) for a in artifacts]))
+          "
+          echo $artifact_json \
+            | python -c "$get_artifact_ids" \
+            | xargs -I@ bash -c "gh api -H 'Accept: application/vnd.github+json' /repos/modflowpy/flopy/actions/artifacts/@/zip >> ./autotest/.benchmarks/@.zip"
+          zipfiles=( ./autotest/.benchmarks/*.zip )
+          if (( ${#zipfiles[@]} )); then
+            unzip -o './autotest/.benchmarks/*.zip' -d ./autotest/.benchmarks
+          fi
+          python ./scripts/process_benchmarks.py ./autotest/.benchmarks ./autotest/.benchmarks
+        env:
+          ARTIFACTS: ${{steps.run_tests.outputs.artifact_ids}}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v2
+        with:
+          name: benchmarks-${{ github.run_id }}
+          path: |
+            ./autotest/.benchmarks/*.csv
+            ./autotest/.benchmarks/*.png