Merge pull request #21 from joerick/environment

CIBW_ENVIRONMENT
Fixes #16

.travis.yml

@@ -8,10 +8,10 @@ matrix:
 script:
   - |
     if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      pip install .
-      python ./run_tests.py
+      pip install -r requirements-dev.txt
+      python ./bin/run_tests.py
     else
       # linux test requires root to clean up the wheelhouse (docker runs as root)
-      sudo pip install .
-      sudo python ./run_tests.py
+      sudo pip install -r requirements-dev.txt
+      sudo python ./bin/run_tests.py
     fi

README.md

@@ -113,24 +113,43 @@ Default: `auto`
 
 For `linux` you need Docker running, on Mac or Linux. For `macos`, you need a Mac machine, and note that this script is going to automatically install MacPython on your system, so don't run on your development machine. For `windows`, you need to run in Windows, and it will build and test for all versions of Python at `C:\PythonXX[-x64]`.
 
-| Environment variable: `CIBW_TEST_COMMAND`
+| Environment variable: `CIBW_SKIP`
 | ---
 
 Optional.
 
-Shell command to run the tests. The project root should be included in the command as "{project}". The wheel will be installed automatically and available for import from the tests.
+Space-separated list of builds to skip. Each build has an identifier like `cp27-manylinux1_x86_64` or `cp34-macosx_10_6_intel` - you can list ones to skip here and `cibuildwheel` won't try to build them.
 
-Example: `nosetests {project}/tests`
+The format is `python_tag-platform_tag`. The tags are as defined in [PEP 0425](https://www.python.org/dev/peps/pep-0425/#details).
 
-| Environment variable: `CIBW_TEST_REQUIRES`
+Python tags look like `cp27` `cp34` `cp35` `cp36`
+
+Platform tags look like `macosx_10_6_intel` `manylinux1_x86_64` `manylinux1_i386` `win32` `win_amd64`
+
+You can also use shell-style globbing syntax (as per `fnmatch`) 
+
+Example: `cp27-macosx_10_6_intel`  (don't build on Python 2 on Mac)  
+Example: `cp27-win*`  (don't build on Python 2.7 on Windows)  
+Example: `cp34-* cp35-*`  (don't build on Python 3.4 or Python 3.5)  
+
+| Environment variable: `CIBW_ENVIRONMENT`
 | ---
 
 Optional.
 
-Space-separated list of dependencies required for running the tests.
+A space-separated list of environment variables to set during the build. Bash syntax should be used (even on Windows!).
 
-Example: `pytest`  
-Example: `nose==1.3.7 moto==0.4.31`
+You must set this variable to pass variables to Linux builds (since they execute in a Docker container). It also works for the other platforms.
+
+You can use `$PATH` syntax to insert other variables, or the `$(pwd)` syntax to insert the output of other shell commands.
+
+Example: `CFLAGS="-g -Wall" CXXFLAGS="-Wall"`  
+Example: `PATH=$PATH:/usr/local/bin`  
+Example: `BUILD_TIME="$(date)"`  
+Example: `PIP_EXTRA_INDEX_URL="https://pypi.myorg.com/simple"`  
+
+Platform-specific variants also available:
+`CIBW_ENVIRONMENT_MACOS` | `CIBW_ENVIRONMENT_WINDOWS` | `CIBW_ENVIRONMENT_LINUX`
 
 | Environment variable: `CIBW_BEFORE_BUILD`
 | ---
@@ -149,24 +168,24 @@ Example: `{pip} install pybind11`
 Platform-specific variants also available:  
  `CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX`
 
-| Environment variable: `CIBW_SKIP`
+| Environment variable: `CIBW_TEST_COMMAND`
 | ---
 
 Optional.
 
-Space-separated list of builds to skip. Each build has an identifier like `cp27-manylinux1_x86_64` or `cp34-macosx_10_6_intel` - you can list ones to skip here and `cibuildwheel` won't try to build them.
+Shell command to run tests after the build. The wheel will be installed automatically and available for import from the tests. The project root should be included in the command as "{project}".
 
-The format is `python_tag-platform_tag`. The tags are as defined in [PEP 0425](https://www.python.org/dev/peps/pep-0425/#details).
+Example: `nosetests {project}/tests`
 
-Python tags look like `cp27` `cp34` `cp35` `cp36`
+| Environment variable: `CIBW_TEST_REQUIRES`
+| ---
 
-Platform tags look like `macosx_10_6_intel` `manylinux1_x86_64` `manylinux1_i386` `win32` `win_amd64`
+Optional.
 
-You can also use shell-style globbing syntax (as per `fnmatch`) 
+Space-separated list of dependencies required for running the tests.
 
-Example: `cp27-macosx_10_6_intel `  (don't build on Python 2 on Mac)  
-Example: `cp27-win*`  (don't build on Python 2.7 on Windows)  
-Example: `cp34-* cp35-*`  (don't build on Python 3.4 or Python 3.5)  
+Example: `pytest`  
+Example: `nose==1.3.7 moto==0.4.31`
 
 --
 

appveyor.yml

@@ -1,5 +1,5 @@
 build_script:
-  - pip install .
+  - pip install -r requirements-dev.txt
   # the '-u' flag is required so the output is in the correct order. 
   # See https://github.com/joerick/cibuildwheel/pull/24 for more info.
-  - python -u ./run_tests.py
+  - python -u ./bin/run_tests.py

bin/dev_run_test

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+cd "$(dirname "$0")"
+cd ..
+
+CIBW_PLATFORM=linux ./bin/run_test.py $1

bin/run_test.py

@@ -0,0 +1,45 @@
+#!/usr/bin/python
+
+from __future__ import print_function
+import os, sys, subprocess, shutil, json
+from glob import glob
+
+def single_run(test_project):
+    # load project settings into environment
+    env_file = os.path.join(test_project, 'environment.json')
+    project_env = {}
+    if os.path.exists(env_file):
+        with open(env_file) as f:
+            project_env = json.load(f)
+
+    # run the build
+    env = os.environ.copy()
+    project_env = {str(k): str(v) for k, v in project_env.items()} # unicode not allowed in env
+    env.update(project_env)
+    print('Building %s with environment %s' % (test_project, project_env))
+    subprocess.check_call(['cibuildwheel', test_project], env=env)
+    wheels = glob('wheelhouse/*.whl')
+    print('%s built successfully. %i wheels built.' % (test_project, len(wheels)))
+
+    # check some wheels were actually built
+    assert len(wheels) >= 4
+
+    # clean up
+    shutil.rmtree('wheelhouse')
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("test_project_dir")
+    args = parser.parse_args()
+
+    project_path = os.path.abspath(args.test_project_dir)
+
+    if not os.path.exists(project_path):
+        print('No test project not found.', file=sys.stderr)
+        exit(2)
+
+    single_run(project_path)
+
+    print('Project built successfully.')

bin/run_tests.py

@@ -0,0 +1,29 @@
+#!/usr/bin/python
+
+from __future__ import print_function
+import os, sys, subprocess, shutil, json
+from glob import glob
+
+if __name__ == '__main__':
+    # move cwd to the project root
+    os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+    ### run the unit tests
+
+    subprocess.check_call(['python', '-m', 'pytest', 'unit_test'])
+
+    ### run the integration tests
+
+    test_projects = glob('test/??_*')
+
+    if len(test_projects) == 0:
+        print('No test projects found. Aborting.', file=sys.stderr)
+        exit(2)
+
+    print('Testing projects:', test_projects)
+
+    run_test_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'run_test.py')
+    for project_path in test_projects:
+        subprocess.check_call([sys.executable, run_test_path, project_path])
+
+    print('%d projects built successfully.' % len(test_projects))

cibuildwheel/__main__.py

@@ -3,6 +3,7 @@
 
 import cibuildwheel
 import cibuildwheel.linux, cibuildwheel.windows, cibuildwheel.macos
+from cibuildwheel.environment import parse_environment, EnvironmentParseError
 from cibuildwheel.util import BuildSkipper
 
 def get_option_from_environment(option_name, platform=None):
@@ -72,6 +73,15 @@ def main():
     project_dir = args.project_dir
     before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform)
     skip_config = os.environ.get('CIBW_SKIP', '')
+    environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform) or ''
+
+    try:
+        environment = parse_environment(environment_config)
+    except (EnvironmentParseError, ValueError) as e:
+        print('cibuildwheel: Malformed environment option "%s"' % environment_config, file=sys.stderr)
+        import traceback
+        traceback.print_exc(None, sys.stderr)
+        exit(2)
 
     skip = BuildSkipper(skip_config)
 
@@ -103,6 +113,7 @@ def main():
         test_requires=test_requires,
         before_build=before_build,
         skip=skip,
+        environment=environment,
     )
 
     print_preamble(platform, build_options)

cibuildwheel/bashlex_eval.py

@@ -0,0 +1,67 @@
+import subprocess, shlex
+from collections import namedtuple
+import bashlex
+
+NodeExecutionContext = namedtuple('NodeExecutionContext', ['environment', 'input'])
+
+def evaluate(value, environment):
+    if not value:
+        # empty string evaluates to empty string
+        # (but trips up bashlex)
+        return ''
+
+    command_node = bashlex.parsesingle(value)
+
+    if len(command_node.parts) != 1:
+        raise ValueError('"%s" has too many parts' % value)
+
+    value_word_node = command_node.parts[0]
+
+    return evaluate_node(
+        value_word_node, 
+        context=NodeExecutionContext(environment=environment, input=value)
+    )
+
+
+def evaluate_node(node, context):
+    if node.kind == 'word':
+        return evaluate_word_node(node, context=context)
+    elif node.kind == 'commandsubstitution':
+        return evaluate_command_node(node.command, context=context)
+    elif node.kind == 'parameter':
+        return evaluate_parameter_node(node, context=context)
+    else:
+        raise ValueError('Unsupported bash construct: "%s"' % node.word)
+
+
+def evaluate_word_node(node, context):
+    word_start = node.pos[0]
+    word_end = node.pos[1]
+    word_string = context.input[word_start:word_end]
+    letters = list(word_string)
+
+    for part in node.parts:
+        part_start = part.pos[0] - word_start
+        part_end = part.pos[1] - word_start
+
+        # Set all the characters in the part to None
+        for i in range(part_start, part_end):
+            letters[i] = None
+
+        letters[part_start] = evaluate_node(part, context=context)
+
+    # remove the None letters and concat
+    value = ''.join(l for l in letters if l is not None)
+
+    # apply bash-like quotes/whitespace treatment
+    return ' '.join(word.strip() for word in shlex.split(value))
+
+
+def evaluate_command_node(node, context):
+    words = [evaluate_node(part, context=context) for part in node.parts]
+    command = ' '.join(words)
+    return subprocess.check_output(shlex.split(command), env=context.environment)
+
+
+def evaluate_parameter_node(node, context):
+    return context.environment.get(node.value, '')

cibuildwheel/environment.py

@@ -0,0 +1,80 @@
+import bashlex
+from . import bashlex_eval
+
+
+class EnvironmentParseError(Exception):
+    pass
+
+
+def parse_environment(env_string):
+    env_items = split_env_items(env_string)
+    assignments = [EnvironmentAssignment(item) for item in env_items]
+    return ParsedEnvironment(assignments=assignments)
+
+
+def split_env_items(env_string):
+    '''Splits space-separated variable assignments into a list of individual assignments.
+
+    >>> split_env_items('VAR=abc')
+    ['VAR=abc']
+    >>> split_env_items('VAR="a string" THING=3')
+    ['VAR="a string"', 'THING=3']
+    >>> split_env_items('VAR="a string" THING=\\'single "quotes"\\'')
+    ['VAR="a string"', 'THING=\\'single "quotes"\\'']
+    >>> split_env_items('VAR="dont \\\\"forget\\\\" about backslashes"')
+    ['VAR="dont \\\\"forget\\\\" about backslashes"']
+    >>> split_env_items('PATH="$PATH;/opt/cibw_test_path"')
+    ['PATH="$PATH;/opt/cibw_test_path"']
+    >>> split_env_items('PATH2="something with spaces"')
+    ['PATH2="something with spaces"']
+    '''
+    if not env_string:
+        return []
+
+    command_node = bashlex.parsesingle(env_string)
+    result = []
+
+    for word_node in command_node.parts:
+        part_string = env_string[word_node.pos[0]:word_node.pos[1]]
+        result.append(part_string)
+
+    return result
+
+
+class EnvironmentAssignment(object):
+    def __init__(self, assignment):
+        name, equals, value = assignment.partition('=')
+        if not equals:
+            raise EnvironmentParseError(assignment)
+        self.name = name
+        self.value = value
+
+    def evaluated_value(self, environment):
+        '''Returns the value of this assignment, as evaluated in the environment'''
+        return bashlex_eval.evaluate(self.value, environment=environment)
+
+    def as_shell_assignment(self):
+        return 'export %s=%s' % (self.name, self.value)
+
+    def __repr__(self):
+        return '%s=%s' % (self.name, self.value)
+
+
+class ParsedEnvironment(object):
+    def __init__(self, assignments):
+        self.assignments = assignments
+
+    def as_dictionary(self, prev_environment):
+        environment = prev_environment.copy()
+
+        for assignment in self.assignments:
+            value = assignment.evaluated_value(environment=environment)
+            environment[assignment.name] = value
+
+        return environment
+
+    def as_shell_commands(self):
+        return [a.as_shell_assignment() for a in self.assignments]
+
+    def __repr__(self):
+        return 'ParsedEnvironment(%r)' % [repr(a) for a in self.assignments]

cibuildwheel/linux.py

@@ -9,7 +9,7 @@
     from pipes import quote as shlex_quote
 
 
-def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip):
+def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip, environment):
     try:
         subprocess.check_call(['docker', '--version'])
     except:
@@ -52,6 +52,8 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
             set -o xtrace
             cd /project
 
+            {environment_exports}
+
             for PYBIN in {pybin_paths}; do
                 # Setup
                 rm -rf /tmp/built_wheel
@@ -106,6 +108,7 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
             before_build=shlex_quote(
                 prepare_command(before_build, python='python', pip='pip') if before_build else ''
             ),
+            environment_exports='\n'.join(environment.as_shell_commands()),
         )
 
         docker_process = subprocess.Popen([