Build modernization (GHA, wheels, setuptools) (#407)

* Move most CI to GitHub Actions
* Build sdist
* Build manylinux1 wheels with libyaml ext (also tested with 2010 and 2014)
* Build MacOS x86_64 wheels with libyaml ext
* Windows wheel builds remain on AppVeyor until we drop 2.7 support in 6.0
* Smoke tests of all post-build artifacts
* Add PEP517/518 build declaration (pyproject.toml with setuptools backend)
* Fully move build to setuptools
* Drop Python 3.5 support
* Declare Python 3.9 support
* Update PyPI metadata now that setuptools lets it flow through

Co-authored-by: Matt Davis <[email protected]>

.appveyor.yml

@@ -10,7 +10,7 @@ image:
 
 environment:
   libyaml_repo_url: https://github.com/yaml/libyaml.git
-  libyaml_refspec: 0.2.2
+  libyaml_refspec: 0.2.5
   PYYAML_TEST_GROUP: all
 
 #  matrix:

.github/workflows/ci.yml

@@ -0,0 +1,236 @@
+---
+
+name: PyYAML CI
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+env:
+  LIBYAML_REPO: https://github.com/yaml/libyaml
+  LIBYAML_REF: '0.2.5'
+jobs:
+  python_sdist:
+    name: pyyaml sdist
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout pyyaml
+        uses: actions/checkout@v2
+
+      - name: install a python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.x
+
+      - name: install build deps
+        run: |
+          python -V
+
+          python -m pip install build
+
+      - name: build sdist
+        run: |
+          export PYYAML_FORCE_CYTHON=1  # we DO want to force Cythoning, at least until 6.0
+          export PYYAML_FORCE_LIBYAML=0  # we don't actually want to build the lib
+
+          python -m build .
+
+          # ensure exactly one artifact was produced
+          shopt -s nullglob
+          DISTFILES=(dist/*.tar.gz)
+          if [[ ${DISTFILES[@]} -ne 1 ]]; then
+            echo "unexpected content in dist dir: $(ls dist/*.tar.gz)"
+            exit 1
+          fi
+
+      - name: test sdist
+        run: |
+          # install some libyaml headers
+          # TODO: should we smoke test the sdist against the libyaml we built?
+          sudo apt update
+          sudo apt install libyaml-dev -y
+
+          # ensure Cython is not present so we use only what's in the sdist
+          python -m pip uninstall Cython -y || true
+
+          # pass no extra args- we should auto-install with libyaml since it's present
+          python -m pip install dist/*.tar.gz -v
+
+          python packaging/build/smoketest.py
+
+      - name: upload sdist artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: dist
+          path: dist/*.tar.gz
+
+
+  linux_libyaml:
+    name: libyaml ${{ matrix.arch }} ${{ matrix.platform }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform:
+          # manylinux1 is forward-compatible to 2010/2014
+          #- manylinux2014
+          #- manylinux2010
+          - manylinux1
+        arch:
+          - x86_64
+    env:
+      DOCKER_IMAGE: quay.io/pypa/${{ matrix.platform }}_${{ matrix.arch }}
+    steps:
+      - name: check cached libyaml state
+        id: cached_libyaml
+        uses: actions/cache@v2
+        with:
+          path: |
+            libyaml
+          key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }}
+
+      - name: checkout pyyaml
+        uses: actions/checkout@v2
+        if: steps.cached_libyaml.outputs.cache-hit != 'true'
+
+      - name: build libyaml
+        run: |
+          docker run --rm -v $(pwd):/io -e LIBYAML_REF -e LIBYAML_REPO --workdir /io "$DOCKER_IMAGE" /io/packaging/build/libyaml.sh
+        if: steps.cached_libyaml.outputs.cache-hit != 'true'
+
+  linux_pyyaml:
+    needs: linux_libyaml
+    name: pyyaml ${{ matrix.arch }} ${{ matrix.platform }} ${{ matrix.python_tag }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform:
+          # so long as manylinux1 container builds work, they're forward-compatible to 2010/2014
+          # - manylinux2014
+          # - manylinux2010
+          - manylinux1
+        arch:
+          - x86_64
+        python_tag:
+          # NB: manylinux >=2014 containers don't have Python 2.7, so we have to use exclude to skip it
+          - cp27-cp27mu
+          - cp36-cp36m
+          - cp37-cp37m
+          - cp38-cp38
+          - cp39-cp39
+#        exclude:
+#          - platform: manylinux2014
+#            arch: x86_64
+#            python_tag: cp27-cp27mu
+    env:
+      AW_PLAT: ${{ matrix.platform }}_${{ matrix.arch }}
+      DOCKER_IMAGE: quay.io/pypa/${{ matrix.platform }}_${{ matrix.arch }}
+      PYTHON_TAG: ${{ matrix.python_tag }}
+      PYYAML_BUILD_WHEELS: 1
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: fetch cached libyaml
+        id: cached_libyaml
+        uses: actions/cache@v2
+        with:
+          path: |
+            libyaml
+          key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }}
+
+      - name: ensure libyaml fetched
+        run: exit 1
+        if: steps.cached_libyaml.outputs.cache-hit != 'true'
+
+      - name: start container
+        run: |
+          docker run --name worker -t -d --rm -v $(pwd):/io "$DOCKER_IMAGE" bash
+
+      - name: build/test/package
+        run: |
+          docker exec -e PYTHON_TAG -e PYYAML_RUN_TESTS -e PYYAML_BUILD_WHEELS -e AW_PLAT --workdir /io worker \
+          /io/packaging/build/manylinux.sh
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: dist
+          path: dist/*.whl
+
+  macos_libyaml:
+    name: libyaml ${{ matrix.arch }} ${{ matrix.platform }}
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      matrix:
+        platform:
+          - macos-10.15
+        arch:
+          - x86_64
+    steps:
+      - name: check cached libyaml state
+        id: cached_libyaml
+        uses: actions/cache@v2
+        with:
+          path: |
+            libyaml
+          key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }}
+
+      - name: checkout pyyaml
+        uses: actions/checkout@v2
+        if: steps.cached_libyaml.outputs.cache-hit != 'true'
+
+      - name: build libyaml
+        env:
+          MACOSX_DEPLOYMENT_TARGET: '10.9'
+        run: |
+          brew install automake coreutils
+          bash ./packaging/build/libyaml.sh
+        if: steps.cached_libyaml.outputs.cache-hit != 'true'
+
+
+  macos_pyyaml:
+    needs: macos_libyaml
+    name: pyyaml ${{ matrix.arch }} ${{ matrix.platform }} ${{ matrix.python_tag }}
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      matrix:
+        platform:
+          - macos-10.15
+        arch:
+          - x86_64
+        python_tag:
+          - cp27*
+          - cp36*
+          - cp37*
+          - cp38*
+          - cp39*
+    steps:
+      - name: checkout pyyaml
+        uses: actions/checkout@v2
+
+      - name: get cached libyaml state
+        id: cached_libyaml
+        uses: actions/cache@v2
+        with:
+          path: |
+            libyaml
+          key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }}
+
+      - name: ensure libyaml fetched
+        run: exit 1
+        if: steps.cached_libyaml.outputs.cache-hit != 'true'
+
+      - name: install a python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.x
+
+      - name: build/test/package
+        env:
+          CIBW_BUILD: ${{ matrix.python_tag }}
+          CIBW_BUILD_VERBOSITY: 1
+        run: |
+          bash ./packaging/build/macos.sh
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: dist
+          path: dist/*.whl

.gitignore

@@ -1,8 +1,15 @@
 # build outputs
 /dist/*
 /build/*
-/ext/_yaml.c
+/lib/PyYAML.egg-info/*
+/lib3/PyYAML.egg-info/*
+/wheelhouse/*
+/yaml/_yaml.c
 MANIFEST
+**/*.so
+**/*.dylib
+**/*.pyd
+
 
 # cached Python binaries
 *.py[cdo]

MANIFEST.in

@@ -1,7 +1,10 @@
-include README LICENSE CHANGES setup.py
+include CHANGES README LICENSE Makefile pyproject.toml setup.py
 recursive-include lib/yaml *.py
+recursive-include lib/_yaml *.py
 recursive-include lib3/yaml *.py
+recursive-include lib3/_yaml *.py
 recursive-include examples *.py *.cfg *.yaml
 recursive-include tests/data *
 recursive-include tests/lib *.py
 recursive-include tests/lib3 *.py
+recursive-include yaml *

lib/_yaml/__init__.py

@@ -0,0 +1,31 @@
+# This is a stub package designed to roughly emulate the _yaml
+# extension module, which previously existed as a standalone module
+# and has been moved into the `yaml` package namespace.
+# It does not perfectly mimic its old counterpart, but should get
+# close enough for anyone who's relying on it even when they shouldn't.
+import yaml
+
+if not yaml.__with_libyaml__:
+    from sys import version_info
+
+    exc = ModuleNotFoundError if version_info >= (3, 6) else ImportError
+    raise exc("No module named '_yaml'")
+else:
+    from yaml._yaml import *
+    import warnings
+    warnings.warn(
+        'The _yaml extension module is now located at yaml._yaml'
+        ' and its location is subject to change.  To use the'
+        ' LibYAML-based parser and emitter, import from `yaml`:'
+        ' `from yaml import CLoader as Loader, CDumper as Dumper`.',
+        DeprecationWarning
+    )
+    del warnings
+    # Don't `del yaml` here because yaml is actually an existing
+    # namespace member of _yaml.
+
+__name__ = '_yaml'
+# If the module is top-level (i.e. not a part of any specific package)
+# then the attribute should be set to ''.
+# https://docs.python.org/3.8/library/types.html
+__package__ = ''

lib/yaml/__init__.py

@@ -8,7 +8,7 @@
 from loader import *
 from dumper import *
 
-__version__ = '5.3.1'
+__version__ = '5.4.0a0'
 
 try:
     from cyaml import *

lib/yaml/cyaml.py

@@ -4,7 +4,7 @@
     'CBaseDumper', 'CSafeDumper', 'CDumper'
 ]
 
-from _yaml import CParser, CEmitter
+from yaml._yaml import CParser, CEmitter
 
 from constructor import *