Merge pull request #15 from BMCV/develop

Update to 0.3.0

.github/workflows/regressiontests.yml

@@ -0,0 +1,77 @@
+name: Regression tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: ['develop']
+    paths:
+      - .github/workflows/regressiontests.yml
+      - superdsm/**
+      - examples/**
+      - superdsm.yml
+      - tests/regression
+      - requirements.txt
+
+jobs:
+
+  regression_tests:
+    name: "Test: ${{ matrix.taskdir }}"
+    timeout-minutes: 1440
+    runs-on: gpuagrohr-01
+    container:
+      image: ubuntu:22.04
+    defaults:
+      run:
+        shell: bash -el {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        taskdir:
+          - U2OS
+          - NIH3T3
+          - GOWT1-1
+          - GOWT1-2
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Clear cache  # otherwise corrupted packages can be reported sometimes
+        run: rm -rf /github/home/conda_pkgs_dir
+
+      - name: Setup Miniconda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniconda-version: 'latest'
+          channels: conda-forge, bioconda, defaults
+          auto-update-conda: true
+          auto-activate-base: false
+          activate-environment: superdsm
+          environment-file: superdsm.yml
+
+      - name: Download image data
+        run: |
+          cd examples
+          python load_data.py
+
+      - name: Run SuperDSM
+        run: |
+            python -m "superdsm.batch" examples --task-dir "${{ matrix.taskdir }}"
+            python -m "superdsm.batch" examples --task-dir "${{ matrix.taskdir }}" --run
+        env:
+          SUPERDSM_INTERMEDIATE_OUTPUT: false
+          SUPERDSM_NUM_CPUS: 20
+
+      - name: Validate results
+        id: validation
+        run: |
+          mkdir "actual_csv"
+          sh "tests/regression/validate-${{ matrix.taskdir }}.sh" "actual_csv"
+
+      - name: Upload artifact
+        if: failure() && steps.validation.outcome != 'success'
+        uses: actions/upload-artifact@v3  ## v4 requires GLIBC_2.28 which is not found on host
+        with:
+          name: Results ${{ matrix.taskdir }}
+          path: actual_csv/${{ matrix.taskdir }}

.github/workflows/testsuite.yml

@@ -0,0 +1,45 @@
+name: Test suite
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: ['develop']
+    paths:
+      - .github/workflows/testsuite.yml
+      - superdsm/**
+      - examples/**
+      - superdsm.yml
+      - tests/**
+      - requirements.txt
+
+jobs:
+
+  run_testsuite:
+    name: Test suite
+    runs-on: gpuagrohr-01
+    container:
+      image: ubuntu:22.04
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Clear cache  # otherwise corrupted packages can be reported sometimes
+        run: rm -rf /github/home/conda_pkgs_dir
+
+      - name: Setup Miniconda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniconda-version: 'latest'
+          channels: conda-forge, bioconda, defaults
+          auto-update-conda: true
+          auto-activate-base: false
+          activate-environment: superdsm
+          environment-file: superdsm.yml
+
+      - name: Run SuperDSM
+        run: python -m "unittest"

.github/workflows/validate_pr.yml

@@ -0,0 +1,73 @@
+name: Validate pull request
+
+on:
+  pull_request:
+
+jobs:
+
+  validate_branches:
+    name: Validate branches
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Validate branches
+        if: github.base_ref == 'master' && github.head_ref != 'develop'
+        run: |
+          echo "Contributions should be made against the develop branch, see README.rst."
+          exit 1
+
+  validate_conditional_checks:
+    name: Validate conditional checks
+    timeout-minutes: 1440
+    runs-on: gpuagrohr-02
+    container:
+      image: ubuntu:22.04
+
+    steps:
+
+      - name: Install dependencies
+        run: |
+          apt-get update
+          apt-get install -y ca-certificates
+    
+      - uses: blend/[email protected]
+        with:
+          interval: 20s
+          timeout: 1440m
+          checks-yaml: |
+            - job: 'Test: U2OS'
+              paths:
+                - .github/workflows/regressiontests.yml
+                - superdsm/**
+                - examples/**
+                - superdsm.yml
+                - tests/regression
+            - job: 'Test: NIH3T3'
+              paths:
+                - .github/workflows/regressiontests.yml
+                - superdsm/**
+                - examples/**
+                - superdsm.yml
+                - tests/regression
+            - job: 'Test: GOWT1-1'
+              paths:
+                - .github/workflows/regressiontests.yml
+                - superdsm/**
+                - examples/**
+                - superdsm.yml
+                - tests/regression
+            - job: 'Test: GOWT1-2'
+              paths:
+                - .github/workflows/regressiontests.yml
+                - superdsm/**
+                - examples/**
+                - superdsm.yml
+                - tests/regression
+            - job: 'Test suite'
+              paths:
+                - .github/workflows/testsuite.yml
+                - superdsm/**
+                - examples/**
+                - superdsm.yml
+                - tests/**

.readthedocs.yaml

@@ -3,7 +3,7 @@ version: 2
 build:
   os: "ubuntu-20.04"
   tools:
-    python: "3.7"
+    python: "3.9"
 
 sphinx:
   fail_on_warning: true

README.rst

@@ -27,7 +27,7 @@ Dependency Version Considerations:
 
 The file *superdsm.yml* specifies the Conda environment required to accurately reproduce the results from our publications. For most of the dependencies (maybe even all), newer versions are also known to work, however, it has been observed that using different versions might yield slightly different results. To enhance consistency, reproducibility, and `FAIRness <https://www.nature.com/articles/s41597-022-01710-x>`_, most dependency versions are thus pinned.
 
-This also concerns BLAS, which is pinned to ``blas==1.0``. As an alternative to using Conda, *requirements.txt* specifies the required *pip* dependencies with pinned versions. However, to the best of our knowledge, it is not possible to request a specific BLAS version in *pip*, meaning that using *pip* instead of Conda is discouraged.
+This also concerns BLAS, which is pinned to ``blas==1.0``. As an alternative to using Conda, *requirements.txt* specifies a minimal set of required *pip* dependencies with pinned versions. However, to the best of our knowledge, it is not possible to request a specific BLAS version in *pip*, meaning that using *pip* instead of Conda is discouraged.
 
 Note that our Conda package from Bioconda allows different dependency versions, because otherwise it would not be possible to use the package with newer versions of Python. Thus, when using our Conda package, keep in mind that sticking to the versions of the dependencies specified in *superdsm.yml* is recommended.
 
@@ -36,10 +36,15 @@ Performance Considerations:
 
 For full performance on both Intel and AMD CPUs, NumPy with MKL support must be used (instead of OpenBLAS which is often the default, see `details <https://stackoverflow.com/questions/62783262/why-is-numpy-with-ryzen-threadripper-so-much-slower-than-xeon>`_). When using Conda, this can be ensured by adding the dependency ``blas=1.0=mkl`` to the Conda environment (or ``blas=*=mkl`` to allow different versions).
 
-To take advantage of the acceleration provided by MKL on AMD CPUs, the MKL version must be pinned to ``2020.0``. Both specifications are included in the Conda environment specified in *superdsm.yml*. In addition, the environment variable ``MKL_DEBUG_CPU_TYPE=5`` must be set.
+To take advantage of the acceleration provided by MKL on AMD CPUs, the MKL version must be pinned to ``2020.0``. Both specifications are included in the Conda environment specified in *superdsm.yml*. In addition, the environment variable ``MKL_DEBUG_CPU_TYPE=5`` must be set. This happens automatically if the CPU is recognized as an AMD CPU. If automatic recognition of the CPU vendor fails, a warning is displayed when the module is loaded, in which case ``MKL_DEBUG_CPU_TYPE=5`` should be set automatically for AMD GPUs.
 
 Later versions of MKL do not support ``MKL_DEBUG_CPU_TYPE=5``, and previous versions do not offer the required APIs. Unfortunately, it looks like this particular version of MKL has been removed from PyPI (see `available versions <https://pypi.org/project/mkl/#history>`_), so it is not possible to gain the full performance on AMD CPUs using *pip* instead of Conda, and thus the version of MKL is not pinned in *requirements.txt* by default.
 
+Contributions:
+""""""""""""""
+
+Contributions should be made against the ``develop`` branch, so that the documentation build on readthedocs.io is triggered, the documentation is built and reviewed (see `here <https://superdsm.readthedocs.io/en/develop/>`_), before ``develop`` is merged into ``master``. This ensures that the ``master`` branch always has an up-to-date documentation.
+
 Publications:
 """""""""""""
 

docs/requirements.txt

@@ -1,2 +1,3 @@
 sphinx-autorun>=1.1.1
 sphinx-rtd-theme
+pillow==10.2  # newer versions seem to require numpy>=1.21 which conflicts with /requirements.txt

docs/source/conf.py

@@ -3,7 +3,7 @@
 # -- Project information
 
 project = 'SuperDSM'
-copyright = '2017-2023 Leonid Kostrykin, Biomedical Computer Vision Group, Heidelberg University'
+copyright = '2017-2024 Leonid Kostrykin, Biomedical Computer Vision Group, Heidelberg University'
 author = 'Leonid Kostrykin'
 
 # -- General configuration

docs/source/usage.rst

@@ -68,6 +68,23 @@ The pipeline data object ``data`` is a dictionary containing all the intermediat
 
 The ``seg`` object returned by the :py:meth:`~superdsm.render.render_result_over_image` function is an RGB image (represented by a ``numpy.ndarray`` object) which can be visualized directly (e.g., using matplotlib) or saved for later use (e.g., using :py:meth:`~superdsm.io.imwrite`). Use :py:meth:`~superdsm.render.rasterize_labels` to obtain segmentation masks from the pipeline data object.
 
+.. _env_variables:
+
+Environment variables
+*********************
+
+MKL_DEBUG_CPU_TYPE
+
+    To take advantage of the acceleration provided by MKL on AMD CPUs, the environment variable ``MKL_DEBUG_CPU_TYPE=5`` should be set when using an AMD CPU. This usually happens automatically, unless automatic recognition of the CPU vendor fails (and a warning is shown).
+
+SUPERDSM_INTERMEDIATE_OUTPUT
+
+   Set ``SUPERDSM_INTERMEDIATE_OUTPUT=0`` to mute the intermediate console output.
+
+SUPERDSM_NUM_CPUS
+
+   Set ``SUPERDSM_NUM_CPUS=8`` to use 8 CPU cores in batch processing. Defaults to 2. Ignored when used interactively.
+
 .. _references:
 
 References

examples/BBBC039/task.json

@@ -1,6 +1,5 @@
 {
     "runnable": true,
-    "num_cpus": 16,
     "environ": {
         "MKL_NUM_THREADS": 2,
         "OPENBLAS_NUM_THREADS": 2

examples/GOWT1-1/task.json

@@ -1,6 +1,5 @@
 {
     "runnable": false,
-    "num_cpus": 16,
     "environ": {
         "MKL_NUM_THREADS": 2,
         "OPENBLAS_NUM_THREADS": 2

examples/GOWT1-2/task.json

@@ -1,6 +1,5 @@
 {
     "runnable": false,
-    "num_cpus": 16,
     "environ": {
         "MKL_NUM_THREADS": 2,
         "OPENBLAS_NUM_THREADS": 2

examples/NIH3T3/task.json

@@ -1,6 +1,5 @@
 {
     "runnable": false,
-    "num_cpus": 16,
     "environ": {
         "MKL_NUM_THREADS": 2,
         "OPENBLAS_NUM_THREADS": 2

examples/U2OS/task.json

@@ -1,6 +1,5 @@
 {
     "runnable": false,
-    "num_cpus": 16,
     "environ": {
         "MKL_NUM_THREADS": 2,
         "OPENBLAS_NUM_THREADS": 2

examples/fibroblast/task.json

@@ -1,6 +1,5 @@
 {
     "runnable": false,
-    "num_cpus": 16,
     "environ": {
         "MKL_NUM_THREADS": 2,
         "OPENBLAS_NUM_THREADS": 2

requirements.txt

@@ -4,7 +4,8 @@ scikit-image==0.18.1
 ipython>=7.31.1
 dill==0.3.2
 ray>=0.8.7
-cvxopt==1.2.6
+cvxopt==1.2.7
 cvxpy==1.1.13
 matplotlib>=3.0
 mkl>=2020.0
+imagecodecs==2022.9.26