Merge remote-tracking branch 'origin/master' into mpi

.github/dependabot.yml

@@ -9,3 +9,7 @@ updates:
       interval: "daily"
     labels:
       - "Bot"
+    groups:
+      github-actions:
+        patterns:
+          - '*'

.github/workflows/cibuildwheel.yml

@@ -0,0 +1,204 @@
+name: Wheels
+
+on:
+  pull_request:
+  push:
+    tags:
+      - "v*"
+  release:
+    types:
+      - published
+
+permissions:
+  contents: read
+
+jobs:
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        name: Install Python
+        with:
+          python-version: 3.x
+
+      - name: Install APT packages
+        if: contains(${{ matrix.os }}, 'ubuntu')
+        run: |
+          sudo apt update
+          sudo apt install libhdf5-dev libnetcdf-dev
+
+      - name: Build sdist
+        run: >
+          pip install build
+          &&  python -m build --sdist . --outdir dist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: pypi-artifacts
+          path: ${{ github.workspace }}/dist/*.tar.gz
+
+
+  build_bdist:
+    name: "Build ${{ matrix.os }} (${{ matrix.arch }}) wheels"
+    runs-on: ${{ matrix.os }}
+    # Prevent hanging when building from emulation like aarch64.
+    timeout-minutes: 300
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-22.04
+            arch: x86_64
+          - os: ubuntu-22.04
+            arch: aarch64
+          - os: macos-14
+            arch: arm64
+            CIBW_ENVIRONMENT: MACOSX_DEPLOYMENT_TARGET=14.0
+          - os: macos-11
+            arch: x86_64
+            CIBW_ENVIRONMENT: MACOSX_DEPLOYMENT_TARGET=11.0
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    # For aarch64 support
+    # https://cibuildwheel.pypa.io/en/stable/faq/#emulation
+    - uses: docker/setup-qemu-action@v3
+      with:
+        platforms: all
+      if: runner.os == 'Linux' && matrix.arch == 'aarch64'
+
+    - name: Build oldest and newest Python
+      shell: bash
+      # On PRs we run only oldest and newest Python versions to reduce CI load.
+      # Skips pypy and musllinux everywhere.
+      # We are buiding 38 and 312 for now.
+      # These needs to rotate every new Python release.
+      run: |
+        if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+          CIBW_SKIP="pp* cp36-* cp37-* *-musllinux* cp39-* cp310-* cp311-*"
+        else
+          CIBW_SKIP="pp* cp36-* cp37-* *-musllinux*"
+        fi
+        echo "CIBW_SKIP=$CIBW_SKIP" >> $GITHUB_ENV
+        echo "Setting CIBW_SKIP=$CIBW_SKIP"
+
+    - name: "Building ${{ matrix.os }} (${{ matrix.arch }}) wheels"
+      uses: pypa/[email protected]
+      env:
+        CIBW_SKIP: ${{ env.CIBW_SKIP }}
+        CIBW_ARCHS: ${{ matrix.arch }}
+        CIBW_BUILD_FRONTEND: build
+        CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/ocefpaf/manylinux2014_x86_64-netcdf
+        CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/ocefpaf/manylinux2014_aarch64-netcdf
+        # Emulation testing is slow, testing only latest Python.
+        CIBW_TEST_SKIP: "cp38-*_aarch64 cp39-*_aarch64 cp310-*_aarch64 cp311-*_aarch64"
+        CIBW_ENVIRONMENT: ${{ matrix.CIBW_ENVIRONMENT }}
+        CIBW_BEFORE_BUILD_MACOS: brew install hdf5 netcdf
+        CIBW_TEST_REQUIRES: pytest cython packaging
+        CIBW_TEST_COMMAND: >
+          python -c "import netCDF4; print(f'netCDF4 v{netCDF4.__version__}')"
+          && pytest -s -rxs -v {project}/test
+          && URL="https://icdc.cen.uni-hamburg.de/thredds/dodsC/ftpthredds/hamtide/m2.hamtide11a.nc"
+          && python -c "from netCDF4 import Dataset; nc=Dataset(\"${URL}\"); print(nc)"
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: pypi-artifacts-${{ matrix.os }}-${{ matrix.arch }}
+        path: ${{ github.workspace }}/wheelhouse/*.whl
+
+
+  build_wheels_windows:
+    name: Build wheels for ${{matrix.arch}} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest]
+        arch: [win_amd64]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        name: Install Python
+        with:
+          python-version: 3.x
+
+      - name: Setup Micromamba Python ${{ matrix.python-version }}
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-name: build
+          init-shell: bash
+          create-args: >-
+            python=${{ matrix.python-version }} libnetcdf=4.9.2 --channel conda-forge
+
+      - name: Install cibuildwheel
+        run: |
+          python -m pip install --upgrade cibuildwheel delvewheel
+
+      - name: Build wheels for Windows (${{ matrix.arch }})
+        run: cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_BUILD: "cp39-${{ matrix.arch }} cp310-${{ matrix.arch }} cp311-${{ matrix.arch }} cp312-${{ matrix.arch }}"
+          CIBW_ENVIRONMENT_WINDOWS: >
+            HDF5_DIR="C:\\Users\\runneradmin\\micromamba\\envs\\build\\Library"
+            netCDF4_DIR="C:\\Users\\runneradmin\\micromamba\\envs\\build\\Library"
+            PATH="C:\\Users\\runneradmin\\micromamba\\envs\\build\\Library\\bin;${PATH}"
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >
+            delvewheel show {wheel}
+            && delvewheel repair -w {dest_dir} {wheel}
+          CIBW_TEST_REQUIRES: pytest cython packaging
+          CIBW_TEST_COMMAND: >
+            python -c "import netCDF4; print(f'netCDF4 v{netCDF4.__version__}')"
+            && pytest -s -rxs -v {project}\\test
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: pypi-artifacts-${{ matrix.os }}-${{ matrix.arch }}
+          path: ${{ github.workspace }}/wheelhouse/*.whl
+
+
+  show-artifacts:
+    needs: [build_bdist, build_sdist, build_wheels_windows]
+    name: "Show artifacts"
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/download-artifact@v4
+      with:
+        pattern: pypi-artifacts*
+        path: ${{ github.workspace }}/dist
+        merge-multiple: true
+
+    - shell: bash
+      run: |
+        ls -lh ${{ github.workspace }}/dist
+
+
+  publish-artifacts-pypi:
+    needs: [build_bdist, build_sdist, build_wheels_windows]
+    name: "Publish to PyPI"
+    runs-on: ubuntu-22.04
+    # upload to PyPI for every tag starting with 'v'
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
+    steps:
+    - uses: actions/download-artifact@v4
+      with:
+        pattern: pypi-artifacts*
+        path: ${{ github.workspace }}/dist
+        merge-multiple: true
+
+    - uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_PASSWORD }}
+        print_hash: true

.github/workflows/miniconda.yml

@@ -12,13 +12,16 @@ jobs:
     #  NO_NET: 1
     strategy:
       matrix:
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
         os: [windows-latest, ubuntu-latest, macos-latest]
         platform: [x64, x32]
         exclude:
           - os: macos-latest  
             platform: x32
       fail-fast: false
+    defaults:
+      run:
+        shell: bash -l {0}
 
     steps:
     - uses: actions/checkout@v4
@@ -36,13 +39,11 @@ jobs:
           --channel conda-forge
 
     - name: Install netcdf4-python
-      shell: bash -l {0}
       run: |
         export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config
-        python -m pip install -v -e . --no-deps --force-reinstall
+        python -m pip install -v -e . --no-deps --no-build-isolation --force-reinstall
 
     - name: Tests
-      shell: bash -l {0}
       run: |
         cd test && python run_all.py
 
@@ -53,6 +54,9 @@ jobs:
         python-version: [ "3.11" ]
         os: [ubuntu-latest]
         platform: [x64]
+    defaults:
+      run:
+        shell: bash -l {0}
     steps:
     - uses: actions/checkout@v4
       with:
@@ -69,22 +73,20 @@ jobs:
           --channel conda-forge
 
     - name: Install netcdf4-python with mpi
-      shell: bash -l {0}
       run: |
         export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config
         nc-config --all
         python -m pip install -v -e . --no-build-isolation --no-deps --force-reinstall
 
     - name: Tests
-      shell: bash -l {0}
       run: |
         cd test && python run_all.py
         cd ../examples
         export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" 
         which mpirun
         mpirun --version
-        #mpirun -np 4 --oversubscribe python mpi_example.py # for openmpi
-        mpirun -np 4 python mpi_example.py
+        mpirun -np 4 --oversubscribe python mpi_example.py # for openmpi
+        #mpirun -np 4 python mpi_example.py
         if [ $? -ne 0 ] ; then
           echo "hdf5 mpi test failed!"
           exit 1

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "external/nc_complex"]
-	path = external/nc_complex
-	url = https://github.com/PlasmaFAIR/nc-complex.git

Changelog

@@ -1,11 +1,19 @@
- version 1.7.0 (not yet released)
+ version 1.7.1 (tag v1.7.1rel)
+===============================
+ * include nc_complex source code from v0.2.0 tag (instead of using submodule).
+ * add aarch64 wheels.
+
+ version 1.7.0 (tag v1.7.0rel)
 ===============================
  * add support for complex numbers via `auto_complex` keyword to `Dataset` (PR #1295)
  * fix for deprecated Cython `DEF` and `IF` statements using compatibility header
    with shims for unavailable functionality (PR #1277)
  * use `szip` as the library name on Windows (PR #1304)
  * add support for MS-MPI `MPI_Message` detection (PR #1305)
-
+ * fix for issue #1306 - surprising result when indexing vlen str with non-contiguous
+   indices.
+ * Fix bug in set_collective introduced in PR #1277 (collective mode was
+   always set).
 
  version 1.6.5 (tag v1.6.5rel)
 ===============================

MANIFEST.in

@@ -1,5 +1,6 @@
 include docs/index.html
 recursive-include man *
+recursive-include external *
 include MANIFEST.in
 include README.htmldocs
 include Changelog
@@ -17,6 +18,9 @@ include src/netCDF4/plugins/empty.txt
 include include/netCDF4.pxi
 include include/mpi-compat.h
 include include/membuf.pyx
+include include/netcdf-compat.h
+include include/no_parallel_support_imports.pxi.in
+include include/parallel_support_imports.pxi.in
 include *.md
 include *.py
 include *.release

README.md

@@ -10,6 +10,10 @@
 ## News
 For details on the latest updates, see the [Changelog](https://github.com/Unidata/netcdf4-python/blob/master/Changelog).
 
+06/17/2024: Version [1.7.1](https://pypi.python.org/pypi/netCDF4/1.7.1) released. Fixes for wheels, no code changes.
+
+06/13/2024: Version [1.7.0](https://pypi.python.org/pypi/netCDF4/1.7.0) released. Add support for complex numbers via `auto_complex` keyword to `Dataset` ([PR #1295](https://github.com/Unidata/netcdf4-python/pull/1295))
+
 10/20/2023: Version [1.6.5](https://pypi.python.org/pypi/netCDF4/1.6.5) released. 
 Fix for issue #1271 (mask ignored if bool MA assinged to uint8 var), 
 support for python 3.12 (removal of python 3.7 support), more

external/README

@@ -0,0 +1 @@
+* 20240616:  remove submodule, include v0.2.0 tag source files (https://github.com/PlasmaFAIR/nc-complex/releases/tag/v0.2.0).

external/nc_complex/include/generated_fallbacks/nc_complex_version.h

@@ -0,0 +1,4 @@
+#define NC_COMPLEX_GIT_SHA1 "37310ed00f3910974bdefefcdfa4787588651f59"
+#define NC_COMPLEX_GIT_VERSION "v0.2.0"
+#define NC_COMPLEX_GIT_STATE "clean"
+#define NC_COMPLEX_GIT_DATE "2023-12-08"