From 5115aa0c18bba1576125268fca32f682f12e5886 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 12:54:39 +0100 Subject: [PATCH 01/45] Add multibuild submodule (master branch) --- .gitmodules | 3 +++ multibuild | 1 + 2 files changed, 4 insertions(+) create mode 160000 multibuild diff --git a/.gitmodules b/.gitmodules index f49812634..25ea5a030 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = tests/data url = https://github.com/adobe-type-tools/psautohint-testdata.git ignore = dirty +[submodule "multibuild"] + path = multibuild + url = https://github.com/matthew-brett/multibuild/ diff --git a/multibuild b/multibuild new file mode 160000 index 000000000..7cca00d3d --- /dev/null +++ b/multibuild @@ -0,0 +1 @@ +Subproject commit 7cca00d3d1d590a3ed34185ef1db1cd970c3a3a1 From 0b5e0e197d6b922743ba25ec8a96e7e7e8758a30 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 14:37:53 +0100 Subject: [PATCH 02/45] remove redundant tests/.coveragerc the root .coveragerc is enough --- tests/.coveragerc | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 tests/.coveragerc diff --git a/tests/.coveragerc b/tests/.coveragerc deleted file mode 100644 index 4285aeb50..000000000 --- a/tests/.coveragerc +++ /dev/null @@ -1,34 +0,0 @@ -[run] -# measure 'branch' coverage in addition to 'statement' coverage -# See: http://coverage.readthedocs.org/en/coverage-4.0.3/branch.html#branch -branch = True -parallel = True - -# list of directories or packages to measure -source = psautohint - -# these are treated as equivalent when combining data -[paths] -source = - ../python/psautohint - /home/travis/build/adobe-type-tools/psautohint/build/*/psautohint - -[report] -# Regexes for lines to exclude from consideration -exclude_lines = - # keywords to use in inline comments to skip coverage - pragma: no cover - - # don't complain if tests don't hit defensive assertion code - raise AssertionError - raise NotImplementedError - - # don't complain if non-runnable code isn't run - if 0: - if __name__ == .__main__.: - -# ignore source code that can’t be found -ignore_errors = True - -# when running a summary report, show missing lines -show_missing = True From 9c4e39bfacbfffe75d944b8d8c11175f69c948d2 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 17:03:09 +0100 Subject: [PATCH 03/45] .gitignore: we store coverage files in .tox, already ignored --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 61108b429..6f50cddc2 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,5 @@ MANIFEST build dist .DS_Store -tests/.coverage* .tox/ .pytest_cache/ From 21f58dbe1b394409483ffae0fa7e8a06feddbe28 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 17:03:46 +0100 Subject: [PATCH 04/45] .gitignore: add htmlcov/ folder --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6f50cddc2..b3ce71d40 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ dist .DS_Store .tox/ .pytest_cache/ +htmlcov/ From 55a8f3c82ded6fc9ee3bed7dea6f1abfdf47d629 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 17:51:39 +0100 Subject: [PATCH 05/45] tests/__init__.py: define DATA_DIR global path to 'tests/data' always relative to the current working directory, so we can run test modules from anywhere --- tests/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index e69de29bb..2986fc76e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,7 @@ +from __future__ import print_function, absolute_import, division +import os + +ROOT = os.path.abspath( + os.path.realpath(os.path.dirname(os.path.dirname(__file__))) +) +DATA_DIR = os.path.relpath(os.path.join(ROOT, "tests", "data"), os.getcwd()) From d8dd8dda5299183b6aa7da769d33be8be35c4b90 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 17:55:29 +0100 Subject: [PATCH 06/45] test_hintfonts: use DATA_DIR global; save output to tmpdir --- tests/test_hintfonts.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_hintfonts.py b/tests/test_hintfonts.py index 72b38455e..afe33c02f 100644 --- a/tests/test_hintfonts.py +++ b/tests/test_hintfonts.py @@ -1,6 +1,7 @@ from __future__ import print_function, division, absolute_import import glob +from os.path import basename import pytest from fontTools.misc.xmlWriter import XMLWriter @@ -8,6 +9,7 @@ from psautohint.autohint import ACOptions, hintFiles from .differ import main as differ +from . import DATA_DIR class Options(ACOptions): @@ -19,18 +21,18 @@ def __init__(self, inpath, outpath): self.verbose = False -@pytest.mark.parametrize("ufo", glob.glob("data/*/*/font.ufo")) -def test_ufo(ufo): - out = ufo + ".out" +@pytest.mark.parametrize("ufo", glob.glob("%s/*/*/font.ufo" % DATA_DIR)) +def test_ufo(ufo, tmpdir): + out = str(tmpdir / basename(ufo)) options = Options(ufo, out) hintFiles(options) assert differ([ufo, out]) -@pytest.mark.parametrize("otf", glob.glob("data/*/*/font.otf")) -def test_otf(otf): - out = otf + ".out" +@pytest.mark.parametrize("otf", glob.glob("%s/*/*/font.otf" % DATA_DIR)) +def test_otf(otf, tmpdir): + out = str(tmpdir / basename(otf)) options = Options(otf, out) hintFiles(options) From af4bfcf1077f5df2947417b7fdc04346167ae67f Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 17:58:24 +0100 Subject: [PATCH 07/45] test_hintfontsmm: reference and outputs in tmpdir, use DATA_DIR py.path is part of pytest (tmpdir fixture itself returns a py.path.local object) --- tests/test_hintfontsmm.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_hintfontsmm.py b/tests/test_hintfontsmm.py index e1a20d3a4..9d2fd236e 100644 --- a/tests/test_hintfontsmm.py +++ b/tests/test_hintfontsmm.py @@ -1,12 +1,17 @@ +from __future__ import print_function, division, absolute_import + import glob import pytest +import py.path from psautohint.autohint import ACOptions, hintFiles from .differ import main as differ +from . import DATA_DIR class Options(ACOptions): + def __init__(self, reference, inpaths, outpaths): super(Options, self).__init__() self.inputPaths = inpaths @@ -16,12 +21,16 @@ def __init__(self, reference, inpaths, outpaths): self.verbose = False -@pytest.mark.parametrize("base", glob.glob("data/*/*Masters")) -def test_mmufo(base): +@pytest.mark.parametrize("base", glob.glob("%s/*/*Masters" % DATA_DIR)) +def test_mmufo(base, tmpdir): paths = sorted(glob.glob(base + "/*.ufo")) - reference = paths[0] + # the reference font is modified in-place, make a temp copy first + referenceSrc = py.path.local(paths[0]) + referenceDst = tmpdir / referenceSrc.basename + referenceSrc.copy(referenceDst) + reference = str(referenceDst) inpaths = paths[1:] - outpaths = [p + ".out" for p in inpaths] + outpaths = [str(tmpdir / p) for p in inpaths] options = Options(reference, inpaths, outpaths) hintFiles(options) From 2044481b9b5f0212ac2b990bac1a3d4971fae0d2 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 18:01:45 +0100 Subject: [PATCH 08/45] .coveragerc: fix source/paths, skip_covered, etc. remove parallel=True as it conficts with pytest-cov and pytest-xdist; add .tox site-packages folders to 'paths' for combining coverage. --- .coveragerc | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.coveragerc b/.coveragerc index 23e567e18..44169ddc5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,14 +2,17 @@ # measure 'branch' coverage in addition to 'statement' coverage # See: http://coverage.readthedocs.io/en/coverage-4.5.1/branch.html branch = True -parallel = True # list of directories or packages to measure +source = psautohint + +# these are treated as equivalent when combining data +[paths] source = python/psautohint - -omit = - */__init__.py + .tox/*/lib/python*/site-packages/psautohint + .tox/*/Lib/site-packages/psautohint + .tox/pypy*/site-packages/psautohint [report] # Regexes for lines to exclude from consideration @@ -18,11 +21,16 @@ exclude_lines = pragma: no cover # don't complain if tests don't hit defensive assertion code - # ... + raise AssertionError + raise NotImplementedError # don't complain if non-runnable code isn't run + if 0: if __name__ == .__main__.: +# Don't include files that are 100% covered +skip_covered = True + # ignore source code that can’t be found ignore_errors = True From 31dd5ed4e1cacf292a638795ae79605effb0e8cb Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 18:19:45 +0100 Subject: [PATCH 09/45] setup.py: add test requirements to 'testing' extra so one can install them, e.g.: pip install -e .[testing] --- setup.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setup.py b/setup.py index 08ec43f0c..798b4cf05 100644 --- a/setup.py +++ b/setup.py @@ -91,6 +91,14 @@ def build_extension(self, ext): install_requires=[ 'fonttools>=3.1.2', ], + extras_require={ + "testing": [ + "pytest >= 3.0.0, <4", + "pytest-cov >= 2.5.1, <3", + "pytest-xdist >= 1.22.2, <2", + "pytest-randomly >= 1.2.3, <2", + ], + }, cmdclass={ 'build_ext': CustomBuildExt, }, From 5cd503da31b5e132fbd0ded33043fd65dcaf317e Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 18:27:18 +0100 Subject: [PATCH 10/45] dev-requirements.txt: only tox; the rest is handled by tox --- dev-requirements.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index ab472a8a9..b898fc96c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1 @@ -# required for formatting the 'long_description' as Markdown -setuptools>=36.4.0 -wheel>=0.31.0 - -coverage>=4.5.1 tox>=2.9.1 From 2f2bf3bba0692857a8c7b253870a21b1e4be1727 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 18:28:15 +0100 Subject: [PATCH 11/45] tox.ini: run in parallel with combined coverage, etc. Several changes, including: - we now use pytest-xdist plugin to distribute tests across several processes, depending on the CPU cores (-n auto) - py37 added to default envlist (but skipped if not present on local developer's machine) - pytest now searches in 'testpaths = tests' when no input is given; runs in verbose mode, with a summary report at the end - coverage data files for each tox environment are saved inside the .tox work dir, then are combined by a specific 'coverage' tox environment which must be run at the end, and which combines and produces reports in different formats: console, xml (for codecov), and html (saved in 'htmlcov') for inspecting locally in a browser. --- tox.ini | 58 +++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/tox.ini b/tox.ini index 2907370a1..4d70e3886 100644 --- a/tox.ini +++ b/tox.ini @@ -1,35 +1,61 @@ [tox] -envlist = py{27,36}-cov +envlist = py{27,36,37}-cov,coverage +minversion = 2.9.1 +skip_missing_interpreters = true [pytest] norecursedirs = data +testpaths = tests +addopts = + -v + -r a + -n auto [testenv] -changedir = tests +description = run the tests with pytest under {basepython} +setenv = + COVERAGE_FILE={toxworkdir}/.coverage.{envname} +deps = -rrequirements.txt +extras = testing commands = - nocov: pytest -v {posargs} - cov: pytest --cov -v {posargs} + nocov: pytest {posargs} + cov: pytest --cov="{envsitepackagesdir}/psautohint" --cov-config={toxinidir}/.coveragerc {posargs} + +[testenv:coverage] +description = run locally after tests to combine coverage data and create reports; + generates a diff coverage against origin/master (or DIFF_AGAINST env var) deps = - pytest - cov: pytest-cov + coverage >= 4.4.1, < 5 + diff_cover +skip_install = true +setenv = + COVERAGE_FILE={toxworkdir}/.coverage +passenv = + DIFF_AGAINST +changedir = {toxinidir} +commands = + coverage erase + coverage combine + coverage report + coverage xml -o {toxworkdir}/coverage.xml + coverage html + diff-cover --compare-branch {env:DIFF_AGAINST:origin/master} {toxworkdir}/coverage.xml [testenv:codecov] -changedir = tests +description = upload coverage data to codecov (depends on 'coverage' running first) +deps = codecov passenv = * -deps = - coverage - codecov skip_install = true ignore_outcome = true -commands = - coverage combine - codecov --env TOXENV +changedir = {toxinidir} +commands = codecov --file "{toxworkdir}/coverage.xml" --env TOXENV [testenv:bdist] -changedir = +description = build sdist and wheel distribution packages to upload to PyPI deps = - setuptools - wheel + setuptools >= 36.4.0 + wheel >= 0.31.0 +changedir = {toxinidir} commands = python setup.py sdist --dist-dir {toxinidir}/dist pip wheel --no-deps --no-index --wheel-dir {toxinidir}/dist --find-links {toxinidir}/dist psautohint From d6627e752740a78ddc794e0b99a5fbd20a4f3eb6 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 18:57:25 +0100 Subject: [PATCH 12/45] tox.ini: split 'sdist' and 'bdist' environments we only need to run tox -e sdist on the CI before uploading to PyPI. I keep 'tox -e bdist' in case one may want to create the wheel locally --- tox.ini | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 4d70e3886..bbb8a4586 100644 --- a/tox.ini +++ b/tox.ini @@ -50,12 +50,22 @@ ignore_outcome = true changedir = {toxinidir} commands = codecov --file "{toxworkdir}/coverage.xml" --env TOXENV -[testenv:bdist] -description = build sdist and wheel distribution packages to upload to PyPI +[testenv:sdist] +description = build sdist to be uploaded to PyPI +skip_install = true deps = setuptools >= 36.4.0 wheel >= 0.31.0 changedir = {toxinidir} commands = - python setup.py sdist --dist-dir {toxinidir}/dist - pip wheel --no-deps --no-index --wheel-dir {toxinidir}/dist --find-links {toxinidir}/dist psautohint + python -c 'import shutil; shutil.rmtree("dist", ignore_errors=True)' + python setup.py sdist --dist-dir dist + +[testenv:bdist] +description = build both sdist and wheel distribution packages +skip_install = true +deps = {[testenv:sdist]deps} +changedir = {toxinidir} +commands = + {[testenv:sdist]commands} + pip wheel --no-deps --no-index --wheel-dir dist --find-links dist psautohint From a808f1c302ecae2bac068a9411f2ef9ae416f393 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 19:00:12 +0100 Subject: [PATCH 13/45] travis: set up multibuild for now only 2.7 and 3.6 (64-bit) for both linux (wide unicode) and mac. No deploy stage yet. --- .travis.yml | 86 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index 61b959ed2..9491d1d9c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,43 +1,65 @@ dist: trusty -sudo: false - +sudo: required language: python +# The travis Python version is unrelated to the version we build and test +# with. This is set with the MB_PYTHON_VERSION variable. +python: 3.5 +services: docker + +branches: + only: + - master + # We want to build wip/* branches since these are not usually used for PRs + - /^wip\/.*$/ + # We want to build version tags as well. + - /^v\d+\.\d+.*$/ + +env: + global: + # directory containing the project source + - REPO_DIR=. + # pip dependencies to _build_ project + # - BUILD_DEPENDS="..." + # pip dependencies to _test_ project + - TEST_DEPENDS="-rdev-requirements.txt" + - PLAT=x86_64 + - UNICODE_WIDTH=32 matrix: + exclude: + # Exclude the default Python 3.5 build + - python: 3.5 include: - - python: 2.7 - env: TOXENV=py27-cov - - python: 3.6 - env: TOXENV=py36-cov - # The 'osx' environment uses the python2.7 by default. - # TODO: maybe also tests python3 on macOS - - language: generic - os: osx - env: TOXENV=py27-cov + - os: linux + env: + - MB_PYTHON_VERSION=2.7 + - os: linux + env: + - MB_PYTHON_VERSION=3.6 + - os: osx + language: generic + env: + - MB_PYTHON_VERSION=2.7 + - os: osx + language: generic + env: + - MB_PYTHON_VERSION=3.6 + +before_install: + - source multibuild/common_utils.sh + - source multibuild/travis_steps.sh + - before_install install: - - if [ "$TRAVIS_OS_NAME" == "osx" ]; then - curl -O https://bootstrap.pypa.io/get-pip.py; - python get-pip.py --user; - python -m pip install --user virtualenv; - python -m virtualenv .venv/; - source .venv/bin/activate; - fi - - pip install --upgrade pip - - pip install -r requirements.txt - - pip install -r dev-requirements.txt + - build_wheel $REPO_DIR $PLAT script: - - tox - - tox -e bdist + - install_run $PLAT after_success: - - tox -e codecov - -branches: - only: - - master - # We want to build wip/* branches since these are not usually used for PRs - - /^wip\/.*$/ - # We want to build version tags as well. - - /^v\d+\.\d+.*$/ + # upload code coverage to Codecov.io + - tox -e coverage,codecov + # if tagged, create the source distribution for the deploy stage + - if [ -n "$TRAVIS_TAG" ] && [ "$BUILD_SDIST" == true ]; then tox -e sdist; fi + # copy compiled wheels to dist/ where Travis `dpl` tool can find them and upload to PyPI + - if [ -n "$TRAVIS_TAG" ]; then mkdir -p dist; cp wheelhouse/*.whl dist; fi From 76bbcc745af242a11ead86c8718b90fb3a9473e9 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 19:18:07 +0100 Subject: [PATCH 14/45] add multibuild's config.sh setup file run_tests will search for the precompiled psautohint wheel inside 'wheelhouse' dir and call tox with --installpkg (to skip rebuilding from sdist). --- config.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 config.sh diff --git a/config.sh b/config.sh new file mode 100644 index 000000000..ff50d293f --- /dev/null +++ b/config.sh @@ -0,0 +1,25 @@ +# Define custom utilities +# Test for OSX with [ -n "$IS_OSX" ] + +function pre_build { + # Any stuff that you need to do before you start building the wheels + # Runs in the root directory of this repository. + : +} + +function run_tests { + # The function is called from an empty temporary directory. + # Get absolute path to the pre-compiled wheel + wheelhouse=$(abspath ../wheelhouse) + wheel=`ls ${wheelhouse}/psautohint*.whl | head -n 1` + if [ ! -e "${wheel}" ]; then + echo "error: can't find wheel in ${wheelhouse} folder" 1>&2 + exit 1 + fi + + # select tox environment based on the current python version + TOXENV="py$(echo ${MB_PYTHON_VERSION} | sed 's/\.//g')-cov" + + # Install pre-compiled wheel and run tests against it + tox --installpkg "${wheel}" -e "${TOXENV}" +} From 83e9e4b8ffdd2661db2b5a03a246711d8cd74096 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 19:30:55 +0100 Subject: [PATCH 15/45] tox.ini: set TOXPYTHON env variable for finding Appveyor pythons --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index bbb8a4586..03365cf66 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,10 @@ addopts = -n auto [testenv] +basepython = + py27: {env:TOXPYTHON:python2.7} + py36: {env:TOXPYTHON:python3.6} + py37: {env:TOXPYTHON:python3.7} description = run the tests with pytest under {basepython} setenv = COVERAGE_FILE={toxworkdir}/.coverage.{envname} From c2a9ccdebfec9ea7ef38783a9fe067ced170b8be Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 19:42:29 +0100 Subject: [PATCH 16/45] tox.ini: add 'wheel' and 'pypi' commands --- tox.ini | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 03365cf66..30901693d 100644 --- a/tox.ini +++ b/tox.ini @@ -65,11 +65,17 @@ commands = python -c 'import shutil; shutil.rmtree("dist", ignore_errors=True)' python setup.py sdist --dist-dir dist -[testenv:bdist] -description = build both sdist and wheel distribution packages +[testenv:wheel] +description = build wheel package for upload to PyPI skip_install = true deps = {[testenv:sdist]deps} changedir = {toxinidir} commands = - {[testenv:sdist]commands} pip wheel --no-deps --no-index --wheel-dir dist --find-links dist psautohint + +[testenv:pypi] +description = upload all distribution files to PyPI (depends on 'sdist' and/or 'wheel') +skip_install = true +deps = twine >= 1.11.0 +changedir = {toxinidir} +commands = twine upload dist/psautohint* From d5d95b80b58a61e4c1653cd6742b1fca705be4e0 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 19:45:50 +0100 Subject: [PATCH 17/45] appveyor: build wheels for both 32 and 64 bit, etc. --- .appveyor.yml | 74 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 8 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index d6c9e485f..f967d532e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,13 +1,57 @@ +cache: + - '%LOCALAPPDATA%\pip\Cache' + environment: matrix: - - TOXENV: py27-cov - - TOXENV: py36-cov + - PYTHON: "C:\\Python27" + PYTHON_VERSION: "2.7" + PYTHON_ARCH: "32" + TOXENV: "py27-cov" + TOXPYTHON: "C:\\Python27\\python.exe" + + - PYTHON: "C:\\Python36" + PYTHON_VERSION: "3.6" + PYTHON_ARCH: "32" + TOXENV: "py36-cov" + TOXPYTHON: "C:\\Python36\\python.exe" + + - PYTHON: "C:\\Python27-x64" + PYTHON_VERSION: "2.7" + PYTHON_ARCH: "64" + TOXENV: "py27-cov" + TOXPYTHON: "C:\\Python27-x64\\python.exe" + + - PYTHON: "C:\\Python36-x64" + PYTHON_VERSION: "3.6" + PYTHON_ARCH: "64" + TOXENV: "py36-cov" + TOXPYTHON: "C:\\Python36-x64\\python.exe" + +matrix: + fast_finish: true + +skip_branch_with_pr: true + +init: + - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" install: - - git submodule update --init - - python -m pip install --upgrade pip - - pip install -r requirements.txt - - pip install -r dev-requirements.txt + # checkout git sub-modules + - git submodule update --init --recursive + + # prepend newly installed Python to the PATH + - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% + + # check that we have the expected version and architecture for Python + - python --version + - python -c "import struct; print(struct.calcsize('P') * 8)" + + # upgrade pip to avoid out-of-date warnings + - python -m pip install --disable-pip-version-check --upgrade pip + - python -m pip --version + + # install tox + - python -m pip install -r dev-requirements.txt build: false @@ -15,7 +59,21 @@ test_script: - tox on_success: - - tox -e codecov + - tox -e coverage,codecov + - > + IF "%APPVEYOR_REPO_TAG%" == "true" + ( + tox -e wheel,pypi + ) -skip_branch_with_pr: true +on_failure: + # print the content of tox *.log files + - ps: Get-Content .tox\\log\\*.log + - ps: $envlogdir = ".tox\\" + ${env:TOXENV} + "\\log"; ` + foreach($logfile in Get-ChildItem $envlogdir){ ` + $logpath = $envlogdir + "\\" + $logfile; ` + Get-Content -path $logpath } +artifacts: + # archive the generated packages in the ci.appveyor.com build report + - path: dist\*.whl From ae22900dd8cbd12be961aaff15c66f07371d0241 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 19:47:23 +0100 Subject: [PATCH 18/45] tox.ini: passenv TWINE_USERNAME and TWINE_PASSWORD --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index 30901693d..770690ddd 100644 --- a/tox.ini +++ b/tox.ini @@ -76,6 +76,9 @@ commands = [testenv:pypi] description = upload all distribution files to PyPI (depends on 'sdist' and/or 'wheel') skip_install = true +passenv = + TWINE_USERNAME + TWINE_PASSWORD deps = twine >= 1.11.0 changedir = {toxinidir} commands = twine upload dist/psautohint* From e6a0ff16c82ee0d8d45f84fd8cba88a21f90c30e Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 20:01:29 +0100 Subject: [PATCH 19/45] tox.ini: must define 'basepython' with TOXPYTHON else Appveyor fails need to tell tox where to find the path to the python interpreter on Windows (do I?) --- tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index 770690ddd..316650a45 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,7 @@ commands = [testenv:coverage] description = run locally after tests to combine coverage data and create reports; generates a diff coverage against origin/master (or DIFF_AGAINST env var) +basepython = {env:TOXPYTHON:python} deps = coverage >= 4.4.1, < 5 diff_cover @@ -47,6 +48,7 @@ commands = [testenv:codecov] description = upload coverage data to codecov (depends on 'coverage' running first) +basepython = {env:TOXPYTHON:python} deps = codecov passenv = * skip_install = true @@ -56,6 +58,7 @@ commands = codecov --file "{toxworkdir}/coverage.xml" --env TOXENV [testenv:sdist] description = build sdist to be uploaded to PyPI +basepython = {env:TOXPYTHON:python} skip_install = true deps = setuptools >= 36.4.0 @@ -67,6 +70,7 @@ commands = [testenv:wheel] description = build wheel package for upload to PyPI +basepython = {env:TOXPYTHON:python} skip_install = true deps = {[testenv:sdist]deps} changedir = {toxinidir} @@ -75,6 +79,7 @@ commands = [testenv:pypi] description = upload all distribution files to PyPI (depends on 'sdist' and/or 'wheel') +basepython = {env:TOXPYTHON:python} skip_install = true passenv = TWINE_USERNAME From 93bf66bf314021258f54b8de9a33c53fd3634e11 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 20:07:40 +0100 Subject: [PATCH 20/45] config.sh: apply codacy suggestions https://app.codacy.com/app/frankrolf/psautohint/pullRequest?prid=1866893 --- config.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/config.sh b/config.sh index ff50d293f..5fe347a81 100644 --- a/config.sh +++ b/config.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # Define custom utilities # Test for OSX with [ -n "$IS_OSX" ] @@ -11,14 +13,15 @@ function run_tests { # The function is called from an empty temporary directory. # Get absolute path to the pre-compiled wheel wheelhouse=$(abspath ../wheelhouse) - wheel=`ls ${wheelhouse}/psautohint*.whl | head -n 1` + wheel=$(ls ${wheelhouse}/psautohint*.whl | head -n 1) if [ ! -e "${wheel}" ]; then echo "error: can't find wheel in ${wheelhouse} folder" 1>&2 exit 1 fi # select tox environment based on the current python version - TOXENV="py$(echo ${MB_PYTHON_VERSION} | sed 's/\.//g')-cov" + # E.g.: '2.7' -> 'py27-cov' + TOXENV="py${MB_PYTHON_VERSION//\./}-cov" # Install pre-compiled wheel and run tests against it tox --installpkg "${wheel}" -e "${TOXENV}" From dcbb5227c1d321325fc34e7103f11c43d6f20018 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 20:13:10 +0100 Subject: [PATCH 21/45] appveyor doesn't like my batch if statement.. let's try powershell --- .appveyor.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index f967d532e..553e49a0e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -60,11 +60,13 @@ test_script: on_success: - tox -e coverage,codecov - - > - IF "%APPVEYOR_REPO_TAG%" == "true" - ( + - ps: >- + if($env:APPVEYOR_REPO_TAG -eq 'true') { + Write-Output ("Deploying " + $env:APPVEYOR_REPO_TAG_NAME + " to PyPI...") tox -e wheel,pypi - ) + } else { + Write-Output "Not deploying as this is not a tagged commit" + } on_failure: # print the content of tox *.log files From 2bb40b7bd9429284b46c66767015cf0be903f619 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 20:16:53 +0100 Subject: [PATCH 22/45] .travis: need to install tox on Travis default python to be able to use it ourside docker container --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9491d1d9c..c2a30a202 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,6 +46,7 @@ matrix: - MB_PYTHON_VERSION=3.6 before_install: + - pip install -r dev-requirements.txt - source multibuild/common_utils.sh - source multibuild/travis_steps.sh - before_install From 2c68fdedde0f318781ca830de05f879659125269 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 20:21:46 +0100 Subject: [PATCH 23/45] tox.ini: need to create sdist to build wheel from it! --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 316650a45..3c6ae81a8 100644 --- a/tox.ini +++ b/tox.ini @@ -75,6 +75,7 @@ skip_install = true deps = {[testenv:sdist]deps} changedir = {toxinidir} commands = + {[testenv:sdist]commands} pip wheel --no-deps --no-index --wheel-dir dist --find-links dist psautohint [testenv:pypi] From a05737ced1731d5062ba5db634369b0a4468d365 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 20:27:04 +0100 Subject: [PATCH 24/45] tox.ini: make 'pypi-wheel' environment; called by Appveyor on tags --- .appveyor.yml | 2 +- tox.ini | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 553e49a0e..3f71a49f3 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -63,7 +63,7 @@ on_success: - ps: >- if($env:APPVEYOR_REPO_TAG -eq 'true') { Write-Output ("Deploying " + $env:APPVEYOR_REPO_TAG_NAME + " to PyPI...") - tox -e wheel,pypi + tox -e pypi-wheel } else { Write-Output "Not deploying as this is not a tagged commit" } diff --git a/tox.ini b/tox.ini index 3c6ae81a8..f8fabdadb 100644 --- a/tox.ini +++ b/tox.ini @@ -78,13 +78,17 @@ commands = {[testenv:sdist]commands} pip wheel --no-deps --no-index --wheel-dir dist --find-links dist psautohint -[testenv:pypi] -description = upload all distribution files to PyPI (depends on 'sdist' and/or 'wheel') +[testenv:pypi-wheel] +description = build and upload wheel package to PyPI basepython = {env:TOXPYTHON:python} skip_install = true passenv = TWINE_USERNAME TWINE_PASSWORD -deps = twine >= 1.11.0 +deps = + {[testenv:wheel]deps} + twine >= 1.11.0 changedir = {toxinidir} -commands = twine upload dist/psautohint* +commands = + {[testenv:wheel]commands} + twine upload dist/psautohint*.whl From b0d90e4746eea571d3c1e39e30d4d1b68c67088d Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 20:32:45 +0100 Subject: [PATCH 25/45] .travis: on osx we need to bootstrap pip in order to install tox... --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index c2a30a202..66261c3f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,6 +46,14 @@ matrix: - MB_PYTHON_VERSION=3.6 before_install: + - | + if [ "$TRAVIS_OS_NAME" == "osx" ]; then + curl -O https://bootstrap.pypa.io/get-pip.py + python get-pip.py --user + python -m pip install --user virtualenv + python -m virtualenv .venv/ + source .venv/bin/activate + fi - pip install -r dev-requirements.txt - source multibuild/common_utils.sh - source multibuild/travis_steps.sh From 58c6132f799c9154ef945996824f2053a854fa56 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 20:39:48 +0100 Subject: [PATCH 26/45] travis: move the osx pip bootstrapping in 'after_success' installing pip seems to break multibuild osx 2.7 build somehow. https://travis-ci.org/adobe-type-tools/psautohint/jobs/400583973 --- .travis.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66261c3f5..5ad9f5d96 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,15 +46,6 @@ matrix: - MB_PYTHON_VERSION=3.6 before_install: - - | - if [ "$TRAVIS_OS_NAME" == "osx" ]; then - curl -O https://bootstrap.pypa.io/get-pip.py - python get-pip.py --user - python -m pip install --user virtualenv - python -m virtualenv .venv/ - source .venv/bin/activate - fi - - pip install -r dev-requirements.txt - source multibuild/common_utils.sh - source multibuild/travis_steps.sh - before_install @@ -66,6 +57,15 @@ script: - install_run $PLAT after_success: + - | + if [ "$TRAVIS_OS_NAME" == "osx" ]; then + curl -O https://bootstrap.pypa.io/get-pip.py + python get-pip.py --user + python -m pip install --user virtualenv + python -m virtualenv .venv/ + source .venv/bin/activate + fi + - pip install -r dev-requirements.txt # upload code coverage to Codecov.io - tox -e coverage,codecov # if tagged, create the source distribution for the deploy stage From b2a91cb2721ac22c2537f7e9dc90932d83860d1d Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 20:42:10 +0100 Subject: [PATCH 27/45] appveyor: disable 2.7 64-bit and 3.6 32-bit.. too slow :( --- .appveyor.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 3f71a49f3..51aeb4abf 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -9,17 +9,17 @@ environment: TOXENV: "py27-cov" TOXPYTHON: "C:\\Python27\\python.exe" - - PYTHON: "C:\\Python36" - PYTHON_VERSION: "3.6" - PYTHON_ARCH: "32" - TOXENV: "py36-cov" - TOXPYTHON: "C:\\Python36\\python.exe" - - - PYTHON: "C:\\Python27-x64" - PYTHON_VERSION: "2.7" - PYTHON_ARCH: "64" - TOXENV: "py27-cov" - TOXPYTHON: "C:\\Python27-x64\\python.exe" + # - PYTHON: "C:\\Python36" + # PYTHON_VERSION: "3.6" + # PYTHON_ARCH: "32" + # TOXENV: "py36-cov" + # TOXPYTHON: "C:\\Python36\\python.exe" + + # - PYTHON: "C:\\Python27-x64" + # PYTHON_VERSION: "2.7" + # PYTHON_ARCH: "64" + # TOXENV: "py27-cov" + # TOXPYTHON: "C:\\Python27-x64\\python.exe" - PYTHON: "C:\\Python36-x64" PYTHON_VERSION: "3.6" From 4caa67012d8ffa361311826d27dc828bfd238102 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 5 Jul 2018 21:15:41 +0100 Subject: [PATCH 28/45] config.sh: remove .tox folder after run_tests to fix permission errors the linux builds fail because they can't read/write the .tox folder.. --- config.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config.sh b/config.sh index 5fe347a81..85111d77a 100644 --- a/config.sh +++ b/config.sh @@ -25,4 +25,8 @@ function run_tests { # Install pre-compiled wheel and run tests against it tox --installpkg "${wheel}" -e "${TOXENV}" + + # clean up after us, or else running tox later on outside the docker + # container can lead to permission errors + rm -rf .tox } From a28434bf97f249f64906899863b0816890f29acf Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 15:34:28 +0100 Subject: [PATCH 29/45] tox.ini: allow to control PYTEST_NUM_PROCESSES via env variable --- tox.ini | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index f8fabdadb..e364619d9 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,6 @@ testpaths = tests addopts = -v -r a - -n auto [testenv] basepython = @@ -22,8 +21,8 @@ setenv = deps = -rrequirements.txt extras = testing commands = - nocov: pytest {posargs} - cov: pytest --cov="{envsitepackagesdir}/psautohint" --cov-config={toxinidir}/.coveragerc {posargs} + nocov: pytest -n {env:PYTEST_NUM_PROCESSES:auto} {posargs} + cov: pytest --cov="{envsitepackagesdir}/psautohint" --cov-config={toxinidir}/.coveragerc -n {env:PYTEST_NUM_PROCESSES:auto} {posargs} [testenv:coverage] description = run locally after tests to combine coverage data and create reports; From 4d76735eb992ba5d54ab85a0630abdb705ccfba7 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 15:55:28 +0100 Subject: [PATCH 30/45] psautohint: always use utf-8 for reading/writing text files When the locale is not set to UTF-8 (like in the manylinux1 docker container, where it's set to 'LANG=C'), some of tests fail. Also, it's safe to assume that UFO xml and plist are alwyas utf-8. https://travis-ci.org/adobe-type-tools/psautohint/jobs/400901533#L663 --- python/psautohint/__main__.py | 2 +- python/psautohint/otfFont.py | 4 ++-- python/psautohint/ufoFont.py | 13 +++++++------ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/python/psautohint/__main__.py b/python/psautohint/__main__.py index 6fd88c66d..faf3a9d5d 100644 --- a/python/psautohint/__main__.py +++ b/python/psautohint/__main__.py @@ -737,7 +737,7 @@ def main(args=None): options = get_options(args) if options.logFile: - open(options.logFile, 'w') + open(options.logFile, 'w', encoding="utf-8") try: hintFiles(options) diff --git a/python/psautohint/otfFont.py b/python/psautohint/otfFont.py index 65d57e369..e1b158d5a 100644 --- a/python/psautohint/otfFont.py +++ b/python/psautohint/otfFont.py @@ -1312,7 +1312,7 @@ def getfdInfo(self, fontPSName, inputPath, allow_no_blues, noFlex, srcFontInfo = os.path.dirname(inputPath) srcFontInfo = os.path.join(srcFontInfo, "fontinfo") if os.path.exists(srcFontInfo): - with open(srcFontInfo, "rU") as fi: + with open(srcFontInfo, "r", encoding="utf-8") as fi: fontInfoData = fi.read() fontInfoData = re.sub(r"#[^\r\n]+", "", fontInfoData) else: @@ -1368,7 +1368,7 @@ def test2(): # Takes first argument = bez path, writes T2 string. # Use form "cid0769" for CID keys references. path = sys.argv[1] - with open(path, "rt") as fp: + with open(path, "rt", encoding="utf-8") as fp: bezString = fp.read() convertBezToT2(bezString) diff --git a/python/psautohint/ufoFont.py b/python/psautohint/ufoFont.py index ac2878fa2..1f9c1180d 100644 --- a/python/psautohint/ufoFont.py +++ b/python/psautohint/ufoFont.py @@ -125,6 +125,7 @@ except ImportError: import xml.etree.ElementTree as ET +from fontTools.misc.py23 import open from psautohint import fdTools @@ -446,7 +447,7 @@ def checkForHints(self, glyphName): hasHints = False glyphPath = self.getGlyphProcessedPath(glyphName) if glyphPath and os.path.exists(glyphPath): - with open(glyphPath, "rt") as fp: + with open(glyphPath, "rt", encoding="utf-8") as fp: data = fp.read() if "hintSetList" in data: hasHints = True @@ -511,7 +512,7 @@ def getGlyphMap(self): def readHashMap(self): hashPath = os.path.join(self.parentPath, "data", kAdobHashMapName) if os.path.exists(hashPath): - with open(hashPath, "rt") as fp: + with open(hashPath, "rt", encoding="utf-8") as fp: data = fp.read() newMap = ast.literal_eval(data) else: @@ -549,7 +550,7 @@ def writeHashMap(self): data.append("}") data.append("") data = "\n".join(data) - with open(hashPath, "wt") as fp: + with open(hashPath, "wt", encoding="utf-8") as fp: fp.write(data) return @@ -996,7 +997,7 @@ def getfdInfo(self, psName, inputPath, allow_no_blues, noFlex, maxY = maxX minY = -self.getUnitsPerEm() if os.path.exists(srcFontInfo): - with open(srcFontInfo, "rU") as fi: + with open(srcFontInfo, "r", encoding="utf-8") as fi: fontInfoData = fi.read() fontInfoData = re.sub(r"#[^\r\n]+", "", fontInfoData) @@ -1221,7 +1222,7 @@ def parsePList(filePath, dictKey=None): # I uses this rather than the plistlib in order # to get a list that allows preserving key order. - with open(filePath, "r") as fp: + with open(filePath, "r", encoding="utf-8") as fp: data = fp.read() contents = XML(data) ufo_dict = contents.find("dict") @@ -2177,7 +2178,7 @@ def convertBezToGLIF(ufoFontData, glyphName, bezString, hintsOnly=False): # I need to replace the contours with data from the bez string. glyphPath = ufoFontData.getGlyphSrcPath(glyphName) - with open(glyphPath, "r") as fp: + with open(glyphPath, "r", encoding="utf-8") as fp: data = fp.read() glifXML = XML(data) From b3cc245b1e313d0519a1111f002c7339ea41992a Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 16:02:48 +0100 Subject: [PATCH 31/45] config.sh: .tox dir to clean up is one dir above run_tests is run from a new temp directory, but .tox dir is always created next to the tox.ini file, in the parent directory in this case --- config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.sh b/config.sh index 85111d77a..ce8e6a3c8 100644 --- a/config.sh +++ b/config.sh @@ -28,5 +28,5 @@ function run_tests { # clean up after us, or else running tox later on outside the docker # container can lead to permission errors - rm -rf .tox + rm -rf ../.tox } From 1c177ee15fdc1921efce2e95458c7135abf985ba Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 16:24:26 +0100 Subject: [PATCH 32/45] ufoFont: fix TypeError when writeHashMap --- python/psautohint/ufoFont.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/python/psautohint/ufoFont.py b/python/psautohint/ufoFont.py index 1f9c1180d..8dd955c7f 100644 --- a/python/psautohint/ufoFont.py +++ b/python/psautohint/ufoFont.py @@ -125,7 +125,7 @@ except ImportError: import xml.etree.ElementTree as ET -from fontTools.misc.py23 import open +from fontTools.misc.py23 import open, tobytes from psautohint import fdTools @@ -550,9 +550,8 @@ def writeHashMap(self): data.append("}") data.append("") data = "\n".join(data) - with open(hashPath, "wt", encoding="utf-8") as fp: - fp.write(data) - return + with open(hashPath, "wb") as fp: + fp.write(tobytes(data, encoding="utf-8")) def getCurGlyphPath(self, glyphName): if self.curSrcDir is None: From 7f07b7e159a03da8fa65ca0d921979f701cbf7a2 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 16:25:18 +0100 Subject: [PATCH 33/45] travis: after 'before_install', we've already sourced a venv --- .travis.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5ad9f5d96..d886fcad6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,15 +57,7 @@ script: - install_run $PLAT after_success: - - | - if [ "$TRAVIS_OS_NAME" == "osx" ]; then - curl -O https://bootstrap.pypa.io/get-pip.py - python get-pip.py --user - python -m pip install --user virtualenv - python -m virtualenv .venv/ - source .venv/bin/activate - fi - - pip install -r dev-requirements.txt + - pip install tox # upload code coverage to Codecov.io - tox -e coverage,codecov # if tagged, create the source distribution for the deploy stage From 7257a039938b98dfbdf52030f7a7ab766c3b1974 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 16:44:52 +0100 Subject: [PATCH 34/45] ufoFont: use ET.parse(path).getroot() instead of ET.XML(string) we already have a source filename; we let elementtree library deal with decoding the text. The XML() factory is for when already starts from a string. And on py2, this must be a byte string... henche this unicodeencodeerror when we attempt to pass a unicode string with non-ascii characters: https://ci.appveyor.com/project/adobe-type-tools/psautohint/build/1.0.237/job/7a4501mbwx2rmqog#L162 --- python/psautohint/ufoFont.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/python/psautohint/ufoFont.py b/python/psautohint/ufoFont.py index 8dd955c7f..ca6d2819a 100644 --- a/python/psautohint/ufoFont.py +++ b/python/psautohint/ufoFont.py @@ -317,7 +317,6 @@ """ -XML = ET.XML XMLElement = ET.Element xmlToString = ET.tostring debug = False @@ -1221,9 +1220,7 @@ def parsePList(filePath, dictKey=None): # I uses this rather than the plistlib in order # to get a list that allows preserving key order. - with open(filePath, "r", encoding="utf-8") as fp: - data = fp.read() - contents = XML(data) + contents = ET.parse(filePath).getroot() ufo_dict = contents.find("dict") if ufo_dict is None: raise UFOParseError("In '%s', failed to find dict. '%s'." % (filePath)) @@ -2176,11 +2173,7 @@ def addWhiteSpace(parent, level): def convertBezToGLIF(ufoFontData, glyphName, bezString, hintsOnly=False): # I need to replace the contours with data from the bez string. glyphPath = ufoFontData.getGlyphSrcPath(glyphName) - - with open(glyphPath, "r", encoding="utf-8") as fp: - data = fp.read() - - glifXML = XML(data) + glifXML = ET.parse(glyphPath).getroot() outlineItem = None libIndex = outlineIndex = -1 From c951a3c72462f23ec4fb62c10bd09279599485cc Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 16:46:47 +0100 Subject: [PATCH 35/45] config.in: cd.. back to repository root before calling tox --- config.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config.sh b/config.sh index ce8e6a3c8..9eb510e9e 100644 --- a/config.sh +++ b/config.sh @@ -11,8 +11,10 @@ function pre_build { function run_tests { # The function is called from an empty temporary directory. + cd .. + # Get absolute path to the pre-compiled wheel - wheelhouse=$(abspath ../wheelhouse) + wheelhouse=$(abspath wheelhouse) wheel=$(ls ${wheelhouse}/psautohint*.whl | head -n 1) if [ ! -e "${wheel}" ]; then echo "error: can't find wheel in ${wheelhouse} folder" 1>&2 @@ -28,5 +30,5 @@ function run_tests { # clean up after us, or else running tox later on outside the docker # container can lead to permission errors - rm -rf ../.tox + rm -rf .tox } From 3c096d79b12fb8e6c064a8b517ff47f51858318b Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 17:08:23 +0100 Subject: [PATCH 36/45] tox: use a single 'codecov' env that inherits from 'coverage' env --- .appveyor.yml | 2 +- .travis.yml | 2 +- tox.ini | 14 +++++++++----- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 51aeb4abf..b29862bfe 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -59,7 +59,7 @@ test_script: - tox on_success: - - tox -e coverage,codecov + - tox -e codecov - ps: >- if($env:APPVEYOR_REPO_TAG -eq 'true') { Write-Output ("Deploying " + $env:APPVEYOR_REPO_TAG_NAME + " to PyPI...") diff --git a/.travis.yml b/.travis.yml index d886fcad6..79bf8be2a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ script: after_success: - pip install tox # upload code coverage to Codecov.io - - tox -e coverage,codecov + - tox -e codecov # if tagged, create the source distribution for the deploy stage - if [ -n "$TRAVIS_TAG" ] && [ "$BUILD_SDIST" == true ]; then tox -e sdist; fi # copy compiled wheels to dist/ where Travis `dpl` tool can find them and upload to PyPI diff --git a/tox.ini b/tox.ini index e364619d9..4c020b64a 100644 --- a/tox.ini +++ b/tox.ini @@ -46,14 +46,18 @@ commands = diff-cover --compare-branch {env:DIFF_AGAINST:origin/master} {toxworkdir}/coverage.xml [testenv:codecov] -description = upload coverage data to codecov (depends on 'coverage' running first) +description = upload coverage data to codecov (only run on CI) basepython = {env:TOXPYTHON:python} -deps = codecov -passenv = * +deps = + {[testenv:coverage]deps} + codecov skip_install = true -ignore_outcome = true +setenv = {[testenv:coverage]setenv} +passenv = * changedir = {toxinidir} -commands = codecov --file "{toxworkdir}/coverage.xml" --env TOXENV +commands = + coverage combine + codecov --env TOXENV [testenv:sdist] description = build sdist to be uploaded to PyPI From 15525101df33f7143a193611d62bf8f0cc6dd07e Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 17:39:24 +0100 Subject: [PATCH 37/45] put .coverage files outside .tox dir at root of repo this is because we remove the whole .tox dir after multibuild docker has completed to avoid permission errors, but then coverage script cannot find the .coverage files --- .gitignore | 1 + tox.ini | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index b3ce71d40..eef1d766b 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ dist .DS_Store .tox/ .pytest_cache/ +.coverage* htmlcov/ diff --git a/tox.ini b/tox.ini index 4c020b64a..a5a044fdc 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ basepython = py37: {env:TOXPYTHON:python3.7} description = run the tests with pytest under {basepython} setenv = - COVERAGE_FILE={toxworkdir}/.coverage.{envname} + COVERAGE_FILE={toxinidir}/.coverage.{envname} deps = -rrequirements.txt extras = testing commands = @@ -33,7 +33,7 @@ deps = diff_cover skip_install = true setenv = - COVERAGE_FILE={toxworkdir}/.coverage + COVERAGE_FILE={toxinidir}/.coverage passenv = DIFF_AGAINST changedir = {toxinidir} @@ -53,7 +53,7 @@ deps = codecov skip_install = true setenv = {[testenv:coverage]setenv} -passenv = * +passenv = TOXENV CI TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* CODECOV_* changedir = {toxinidir} commands = coverage combine From 4998b48df9c5ffa7c7a18e9b5af5f10723a0cc3e Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 18:24:48 +0100 Subject: [PATCH 38/45] travis: add deploy stage; save pip cache --- .travis.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.travis.yml b/.travis.yml index 79bf8be2a..5b53717a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,6 +45,8 @@ matrix: env: - MB_PYTHON_VERSION=3.6 +cache: pip + before_install: - source multibuild/common_utils.sh - source multibuild/travis_steps.sh @@ -64,3 +66,18 @@ after_success: - if [ -n "$TRAVIS_TAG" ] && [ "$BUILD_SDIST" == true ]; then tox -e sdist; fi # copy compiled wheels to dist/ where Travis `dpl` tool can find them and upload to PyPI - if [ -n "$TRAVIS_TAG" ]; then mkdir -p dist; cp wheelhouse/*.whl dist; fi + +deploy: + provider: pypi + on: + tags: true + repo: adobe-type-tools/psautohint + all_branches: true + user: anthrotype + password: + secure: ... # encrypt password with `travis encrypt` + # 'check' is a do-nothing setup.py command, to prevent Travis from trying + # to rebuild the wheel we just built with multibuild + distributions: check + skip_cleanup: true + skip_upload_docs: true From 632b68cf8dd15cc838a08e171b48de46eee184c1 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 18:26:41 +0100 Subject: [PATCH 39/45] travis: add encrypted pypi password --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5b53717a4..88cd410a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -75,7 +75,7 @@ deploy: all_branches: true user: anthrotype password: - secure: ... # encrypt password with `travis encrypt` + secure: NMFwzW1lWIqAxnfkSJ147zPncATi2ToDBpsu1osdxkZafndSUYJGJ1Es1oBnBkmtc73f2G1n9ZoLd3vsL/YhU5VmNJAy3mADgRtpUGAwth5CeHXSyjVNh5nSfjkr2fO3VLMnmHKbhWRHa5OjHTwr341bAIh3Ij10mCiw2Fk26IZIZ9qnp2HP43gsmCNVag0Bp7MhEXiH+mJ2d2R5T/4RNeh1TpQq2NRn5/dIHPX5Y1YAxRF1C0wqu6o3Vyl8NwmS16nH0u7KtGg/hYzFO1s+DdeewRBMzy6v3AZ8faBgsolXMN96i2gVZeAoosCihWflRkkmNioYnPPHWItZFNc7yVv1GMrw1C2dezM9LnpAPKjHFbPm4TahM4sCitgmFIwxFf4Oh8+49rZj75WxnmYCJ/lQWyd84LQG32vj7hnFi/fDV1GVZ+PfrAL6Q/kk1natKSBn9Ue6l3a1uXnzk0/aQR/JuBz+7+Q0uX/4JGcB6QpZRL7KsojinHlb1j8ijuUrbFTepos8IC4Tc5z3GdJxaxbX3F+wNLIfB1p1KMbjb3/vjJSUX6BZ10SOYqCtsxJaQjcfBbZgGtf1KlhJAuDDpwwlFseqi87hyrhK2hFaqq0ha7yD/jXcDrTF0DRDooFNVSiDadtCU8lrfr05Ue5emw/fbjgRzMtPRiNBTQs5IAw= # 'check' is a do-nothing setup.py command, to prevent Travis from trying # to rebuild the wheel we just built with multibuild distributions: check From 35b6813265903a002e5da03ae2691a5048714e31 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 18:33:57 +0100 Subject: [PATCH 40/45] appveyor: add same TWINE_USERNAME and TWINE_PASSWORD as afdko repo I think these should work... --- .appveyor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index b29862bfe..29cadd060 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -2,6 +2,11 @@ cache: - '%LOCALAPPDATA%\pip\Cache' environment: + global: + TWINE_USERNAME: "afdko-travis" + TWINE_PASSWORD: + secure: myMd6E3niwjGvxEHeChOlg== + matrix: - PYTHON: "C:\\Python27" PYTHON_VERSION: "2.7" From 87e1dcb2be1b2ddabefa7f8dab157477cd965675 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 18:50:03 +0100 Subject: [PATCH 41/45] tests/__init__: minor --- tests/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 2986fc76e..d804f5abe 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,5 @@ from __future__ import print_function, absolute_import, division import os -ROOT = os.path.abspath( - os.path.realpath(os.path.dirname(os.path.dirname(__file__))) -) -DATA_DIR = os.path.relpath(os.path.join(ROOT, "tests", "data"), os.getcwd()) +here = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) +DATA_DIR = os.path.relpath(os.path.join(here, "data"), os.getcwd()) From 40741f785cae5742b6ea563490e9367fcfbe6634 Mon Sep 17 00:00:00 2001 From: Miguel Sousa Date: Fri, 6 Jul 2018 11:59:55 -0700 Subject: [PATCH 42/45] [ci-travis] Configure twine [skip ci] --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 88cd410a0..aafe61fa7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,8 @@ env: - TEST_DEPENDS="-rdev-requirements.txt" - PLAT=x86_64 - UNICODE_WIDTH=32 + - TWINE_USERNAME=adobe-type-tools-ci + # TWINE_PASSWORD is set in Travis settings matrix: exclude: @@ -73,9 +75,6 @@ deploy: tags: true repo: adobe-type-tools/psautohint all_branches: true - user: anthrotype - password: - secure: NMFwzW1lWIqAxnfkSJ147zPncATi2ToDBpsu1osdxkZafndSUYJGJ1Es1oBnBkmtc73f2G1n9ZoLd3vsL/YhU5VmNJAy3mADgRtpUGAwth5CeHXSyjVNh5nSfjkr2fO3VLMnmHKbhWRHa5OjHTwr341bAIh3Ij10mCiw2Fk26IZIZ9qnp2HP43gsmCNVag0Bp7MhEXiH+mJ2d2R5T/4RNeh1TpQq2NRn5/dIHPX5Y1YAxRF1C0wqu6o3Vyl8NwmS16nH0u7KtGg/hYzFO1s+DdeewRBMzy6v3AZ8faBgsolXMN96i2gVZeAoosCihWflRkkmNioYnPPHWItZFNc7yVv1GMrw1C2dezM9LnpAPKjHFbPm4TahM4sCitgmFIwxFf4Oh8+49rZj75WxnmYCJ/lQWyd84LQG32vj7hnFi/fDV1GVZ+PfrAL6Q/kk1natKSBn9Ue6l3a1uXnzk0/aQR/JuBz+7+Q0uX/4JGcB6QpZRL7KsojinHlb1j8ijuUrbFTepos8IC4Tc5z3GdJxaxbX3F+wNLIfB1p1KMbjb3/vjJSUX6BZ10SOYqCtsxJaQjcfBbZgGtf1KlhJAuDDpwwlFseqi87hyrhK2hFaqq0ha7yD/jXcDrTF0DRDooFNVSiDadtCU8lrfr05Ue5emw/fbjgRzMtPRiNBTQs5IAw= # 'check' is a do-nothing setup.py command, to prevent Travis from trying # to rebuild the wheel we just built with multibuild distributions: check From c46ed0c8dc77fbf6abe46a70fba33cb3ae9844d0 Mon Sep 17 00:00:00 2001 From: Miguel Sousa Date: Fri, 6 Jul 2018 12:02:16 -0700 Subject: [PATCH 43/45] [ci-appveyor] Configure twine --- .appveyor.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 29cadd060..983be2b36 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -3,9 +3,8 @@ cache: environment: global: - TWINE_USERNAME: "afdko-travis" - TWINE_PASSWORD: - secure: myMd6E3niwjGvxEHeChOlg== + TWINE_USERNAME: "adobe-type-tools-ci" + # TWINE_PASSWORD is set in Appveyor settings matrix: - PYTHON: "C:\\Python27" From 3dcf59a40dbfffb7967419cb6467eab71433fad8 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 20:15:48 +0200 Subject: [PATCH 44/45] appveyor: compile wheel first, then run tests; reuse same wheel for PyPI --- .appveyor.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 983be2b36..c30a6a4d5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -57,17 +57,25 @@ install: # install tox - python -m pip install -r dev-requirements.txt + # build psautohint wheel + - tox -e wheel + - for /f "usebackq tokens=*" %w in (`dir /b dist\psautohint*.whl`) do set WHEEL_PATH=%cd%\dist\%w + build: false test_script: - - tox + # run tests on the compiled wheel + - tox --installpkg "%WHEEL_PATH%" on_success: + # upload test coverage to codecov.io - tox -e codecov + # if commit is tagged, upload wheel to PyPI - ps: >- if($env:APPVEYOR_REPO_TAG -eq 'true') { Write-Output ("Deploying " + $env:APPVEYOR_REPO_TAG_NAME + " to PyPI...") - tox -e pypi-wheel + pip install --upgrade twine + twine upload $env:WHEEL_PATH } else { Write-Output "Not deploying as this is not a tagged commit" } From 0ab12f537856410851e0ebecac6e07f1b25025bd Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 6 Jul 2018 20:16:55 +0200 Subject: [PATCH 45/45] tox.ini: remove unused 'pypi-wheel' env; we call twine directly now --- tox.ini | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tox.ini b/tox.ini index a5a044fdc..2ae54e918 100644 --- a/tox.ini +++ b/tox.ini @@ -80,18 +80,3 @@ changedir = {toxinidir} commands = {[testenv:sdist]commands} pip wheel --no-deps --no-index --wheel-dir dist --find-links dist psautohint - -[testenv:pypi-wheel] -description = build and upload wheel package to PyPI -basepython = {env:TOXPYTHON:python} -skip_install = true -passenv = - TWINE_USERNAME - TWINE_PASSWORD -deps = - {[testenv:wheel]deps} - twine >= 1.11.0 -changedir = {toxinidir} -commands = - {[testenv:wheel]commands} - twine upload dist/psautohint*.whl