From 9141e900d10d280173335b20207e87d4a15e62c6 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Tue, 12 Mar 2019 11:35:50 -0700 Subject: [PATCH] Windows Appveyor build * builds Windows wheels against a specified libyaml repo/refspec for many Python versions * since we don't have multiple Appveyor workers, it's faster/more convenient to run them serially * not all paths sufficient for general CI usage yet; still needs manual inspection/testing of output * various hacks to quiet warning noise during build on old Pythons --- .appveyor.yml | 35 +++++++++ .gitignore | 19 +++-- packaging/build/FixVS9CMake.reg | 76 ++++++++++++++++++ packaging/build/appveyor.ps1 | 134 ++++++++++++++++++++++++++++++++ setup.py | 15 +++- 5 files changed, 270 insertions(+), 9 deletions(-) create mode 100644 .appveyor.yml create mode 100644 packaging/build/FixVS9CMake.reg create mode 100644 packaging/build/appveyor.ps1 diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 00000000..bdad5f31 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,35 @@ +# TODO: update this from inside the build to use branch current version +version: '{build}' + +image: +- Visual Studio 2015 + +#cache: +#- 'C:\Python38\' +#- 'C:\Python38-x64' + +environment: + libyaml_repo_url: https://github.com/yaml/libyaml.git + libyaml_refspec: release/0.2.2 +# matrix: +# - PYTHON_VER: Python27 +# - PYTHON_VER: Python27-x64 +# - PYTHON_VER: Python34 +# - PYTHON_VER: Python34-x64 +# - PYTHON_VER: Python35 +# - PYTHON_VER: Python35-x64 +# - PYTHON_VER: Python36 +# - PYTHON_VER: Python36-x64 +# - PYTHON_VER: Python37 +# - PYTHON_VER: Python37-x64 +# - PYTHON_VER: Python38 +# - PYTHON_VER: Python38-x64 + +#init: +#- ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + +build_script: +- ps: packaging\build\appveyor.ps1 + +#on_finish: +#- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) diff --git a/.gitignore b/.gitignore index 9915308f..4bf45542 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,13 @@ -# This file is generated by Cython -ext/_yaml.c - -# Tox and Build Artifacts -.tox/ +# build outputs +/dist/* +/build/* +/ext/_yaml.c MANIFEST -build/ -# Python runtime Artifacts -*.pyc +# cached Python binaries +*.py[cdo] +_pycache_/* + +# local IDE state +/.idea/* + diff --git a/packaging/build/FixVS9CMake.reg b/packaging/build/FixVS9CMake.reg new file mode 100644 index 00000000..51c444bd --- /dev/null +++ b/packaging/build/FixVS9CMake.reg @@ -0,0 +1,76 @@ +Windows Registry Editor Version 5.00 + +[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{600dd186-2429-11d7-8bf6-00b0d03daa06}] +"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectIA64Platform.dll" +@="Win64 (Itanium) Platform Class" + +[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{600dd187-2429-11d7-8bf6-00b0d03daa06}] +"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectIA64Platform.dll" +@="ClIA64CodeGeneration Class" + +[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{600dd188-2429-11d7-8bf6-00b0d03daa06}] +"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectIA64Platform.dll" +@="ClIA64General Class" + +[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{600dd189-2429-11d7-8bf6-00b0d03daa06}] +"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectIA64Platform.dll" +@="ClIA64AdditionalOptions Class" + +[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{656d875f-2429-11d7-8bf6-00b0d03daa06}] +"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectAMD64Platform.dll" +@="ClAMD64CodeGeneration Class" + +[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{656d8760-2429-11d7-8bf6-00b0d03daa06}] +"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectAMD64Platform.dll" +@="ClAMD64General Class" + +[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{656d8763-2429-11d7-8bf6-00b0d03daa06}] +"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectAMD64Platform.dll" +@="Win64 (AMD64) Platform Class" + +[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{656d8766-2429-11d7-8bf6-00b0d03daa06}] +"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectAMD64Platform.dll" +@="ClAMD64AdditionalOptions Class" + +[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\VC\VC_OBJECTS_PLATFORM_INFO] + +[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\VC\VC_OBJECTS_PLATFORM_INFO\Win64 (AMD64)] +@="{656d8763-2429-11d7-8bf6-00b0d03daa06}" + +[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\VC\VC_OBJECTS_PLATFORM_INFO\Win64 (AMD64)\ToolDefaultExtensionLists] +"VCCLCompilerTool"="*.cpp;*.cxx;*.cc;*.c" +"VCLinkerTool"="*.obj;*.res;*.lib;*.rsc" +"VCLibrarianTool"="*.obj;*.res;*.lib;*.rsc" +"VCMIDLTool"="*.idl;*.odl" +"VCCustomBuildTool"="*.bat" +"VCResourceCompilerTool"="*.rc" +"VCPreBuildEventTool"="*.bat" +"VCPreLinkEventTool"="*.bat" +"VCPostBuildEventTool"="*.bat" +"VCBscMakeTool"="*.sbr" +"VCNMakeTool"="" +"VCWebServiceProxyGeneratorTool"="*.sdl;*.wsdl" +"VCWebDeploymentTool"="" +"VCALinkTool"="*.resources" +"VCManagedResourceCompilerTool"="*.resx" + +[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\VC\VC_OBJECTS_PLATFORM_INFO\Win64 (Itanium)] +@="{600dd186-2429-11d7-8bf6-00b0d03daa06}" + +[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\VC\VC_OBJECTS_PLATFORM_INFO\Win64 (Itanium)\ToolDefaultExtensionLists] +"VCCLCompilerTool"="*.cpp;*.cxx;*.cc;*.c" +"VCLinkerTool"="*.obj;*.res;*.lib;*.rsc" +"VCLibrarianTool"="*.obj;*.res;*.lib;*.rsc" +"VCMIDLTool"="*.idl;*.odl" +"VCCustomBuildTool"="*.bat" +"VCResourceCompilerTool"="*.rc" +"VCPreBuildEventTool"="*.bat" +"VCPreLinkEventTool"="*.bat" +"VCPostBuildEventTool"="*.bat" +"VCBscMakeTool"="*.sbr" +"VCNMakeTool"="" +"VCWebServiceProxyGeneratorTool"="*.sdl;*.wsdl" +"VCWebDeploymentTool"="" +"VCALinkTool"="*.resources" +"VCManagedResourceCompilerTool"="*.resx" + diff --git a/packaging/build/appveyor.ps1 b/packaging/build/appveyor.ps1 new file mode 100644 index 00000000..a9e6ad28 --- /dev/null +++ b/packaging/build/appveyor.ps1 @@ -0,0 +1,134 @@ +# TODO: run in PR/test mode (larger matrix) vs "all-in-one" artifact/packaging mode +# TODO: use dynamic matrix so PRs are multi-job and tag builds are one (consolidate artifacts) +# TODO: consider secure credential storage for inline upload on tags? Or keep that all manual/OOB for security... +# TODO: refactor libyaml/pyyaml tests to enable first-class output for AppVeyor +# TODO: get version number from setup.py and/or lib(3)/__version__ +# Update-AppveyorBuild -Version $dynamic_version + +Function Bootstrap() { + # uncomment when we want to start testing on Python 3.8 + # ensure py38 is present (current Appveyor VS2015 image doesn't include it) + #If(-not $(Test-Path C:\Python38)) { + # choco.exe install python3 --version=3.8.0-a2 --forcex86 --force #--install-arguments="TargetDir=C:\Python38 PrependPath=0" --no-progress + #} + + #If(-not $(Test-Path C:\Python38-x64)) { + # choco.exe install python3 --version=3.8.0-a2 --force #--install-arguments="TargetDir=C:\Python38-x64 PrependPath=0" --no-progress + #} + + Write-Output "patching Windows SDK bits for distutils" + + # patch 7.0/7.1 vcvars SDK bits up to work with distutils query + Set-Content -Path 'C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\amd64\vcvarsamd64.bat' '@CALL "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars64.bat"' + Set-Content -Path 'C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64\vcvars64.bat' '@CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /Release /x64' + + # patch VS9 x64 CMake config for VS Express, hide `reg.exe` stderr noise + $noise = reg.exe import packaging\build\FixVS9CMake.reg 2>&1 + + If($LASTEXITCODE -ne 0) { + throw "reg failed with error code $LASTEXITCODE" + } + + Copy-Item -Path "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\AMD64.VCPlatform.config" -Destination "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\AMD64.VCPlatform.Express.config" -Force + Copy-Item -Path "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\Itanium.VCPlatform.config" -Destination "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\Itanium.VCPlatform.Express.config" -Force + + # git spews all over stderr unless we tell it not to + $env:GIT_REDIRECT_STDERR="2>&1"; + + $libyaml_repo_url = If($env:libyaml_repo_url) { $env:libyaml_repo_url } Else { "https://github.com/yaml/libyaml.git" } + $libyaml_refspec = If($env:libyaml_refspec) { $env:libyaml_refspec } Else { "master" } + + Write-Output "cloning libyaml from $libyaml_repo_url / $libyaml_refspec" + + If(-not $(Test-Path .\libyaml)) { + git clone -b $libyaml_refspec $libyaml_repo_url 2>&1 + } +} + +Function Build-Wheel($python_path) { + + #$python_path = Join-Path C:\ $env:PYTHON_VER + $python = Join-Path $python_path python.exe + + Write-Output "building pyyaml wheel for $python_path" + + # query distutils for the VC version used to build this Python; translate to a VS version to choose the right generator + $python_vs_buildver = & $python -c "from distutils.version import LooseVersion; from distutils.msvc9compiler import get_build_version; print(LooseVersion(str(get_build_version())).version[0])" + + $python_cmake_generator = switch($python_vs_buildver) { + "9" { "Visual Studio 9 2008" } + "10" { "Visual Studio 10 2010" } + "14" { "Visual Studio 14 2015" } + default { throw "Python was built with unknown VS build version: $python_vs_buildver" } + } + + # query arch this python was built for + $python_arch = & $python -c "from distutils.util import get_platform; print(str(get_platform()))" + + if($python_arch -eq 'win-amd64') { + $python_cmake_generator += " Win64" + $vcvars_arch = "x64" + } + + # snarf VS vars (paths, etc) for the matching VS version and arch that built this Python + $raw_vars_out = & cmd.exe /c "`"C:\Program Files (x86)\Microsoft Visual Studio $($python_vs_buildver).0\VC\vcvarsall.bat`" $vcvars_arch & set" + foreach($kv in $raw_vars_out) { + If($kv -match "=") { + $kv = $kv.Split("=", 2) + Set-Item -Force "env:$kv[0]" $kv[1] + } + Else { + Write-Output $kv + } + } + + # ensure pip is current (some appveyor pips are not) + & $python -W "ignore:DEPRECATION" -m pip install --upgrade pip + + # ensure required-for-build packages are present and up-to-date + & $python -W "ignore:DEPRECATION" -m pip install --upgrade cython wheel setuptools --no-warn-script-location + + pushd libyaml + git clean -fdx + popd + + mkdir libyaml\build + + pushd libyaml\build + cmake.exe -G $python_cmake_generator -DYAML_STATIC_LIB_NAME=yaml .. + cmake.exe --build . --config Release + popd + + & $python setup.py --with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build test bdist_wheel +} + +Function Upload-Artifacts() { + Write-Output "uploading artifacts..." + + foreach($wheel in @(Resolve-Path dist\*.whl)) { + Push-AppveyorArtifact $wheel + } +} + +Bootstrap + +$pythons = @( +"C:\Python27" +"C:\Python27-x64" +"C:\Python34" +"C:\Python34-x64" +"C:\Python35" +"C:\Python35-x64" +"C:\Python36" +"C:\Python36-x64" +"C:\Python37" +"C:\Python37-x64" +) + +#$pythons = @("C:\$($env:PYTHON_VER)") + +foreach($python in $pythons) { + Build-Wheel $python +} + +Upload-Artifacts diff --git a/setup.py b/setup.py index 9217435c..2d1c8124 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ """ -import sys, os.path, platform +import sys, os.path, platform, warnings from distutils import log from distutils.core import setup, Command @@ -93,6 +93,19 @@ bdist_wheel = None +# on Windows, disable wheel generation warning noise +windows_ignore_warnings = [ +"Unknown distribution option: 'python_requires'", +"Config variable 'Py_DEBUG' is unset", +"Config variable 'WITH_PYMALLOC' is unset", +"Config variable 'Py_UNICODE_SIZE' is unset", +"Cython directive 'language_level' not set" +] + +if platform.system() == 'Windows': + for w in windows_ignore_warnings: + warnings.filterwarnings('ignore', w) + class Distribution(_Distribution): def __init__(self, attrs=None):