CIBW_ENVIRONMENT

.travis.yml

@@ -8,10 +8,10 @@ matrix:
 script:
   - |
     if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      pip install .
-      python ./run_tests.py
+      pip install -r requirements-dev.txt
+      python ./bin/run_tests.py
     else
       # linux test requires root to clean up the wheelhouse (docker runs as root)
-      sudo pip install .
-      sudo python ./run_tests.py
+      sudo pip install -r requirements-dev.txt
+      sudo python ./bin/run_tests.py
     fi

appveyor.yml

@@ -1,5 +1,5 @@
 build_script:
-  - pip install .
+  - pip install -r requirements-dev.txt
   # the '-u' flag is required so the output is in the correct order. 
   # See https://github.com/joerick/cibuildwheel/pull/24 for more info.
-  - python -u ./run_tests.py
+  - python -u ./bin/run_tests.py

bin/dev_run_test

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+cd "$(dirname "$0")"
+cd ..
+
+CIBW_PLATFORM=linux ./bin/run_test.py $1

bin/run_test.py

@@ -0,0 +1,48 @@
+#!/usr/bin/python
+
+from __future__ import print_function
+import os, sys, subprocess, shutil, json
+from glob import glob
+
+def single_run(test_project):
+    # load project settings into environment
+    env_file = os.path.join(test_project, 'environment.json')
+    project_env = {}
+    if os.path.exists(env_file):
+        with open(env_file) as f:
+            project_env = json.load(f)
+
+    # run the build
+    env = os.environ.copy()
+    project_env = {str(k): str(v) for k, v in project_env.items()} # unicode not allowed in env
+    env.update(project_env)
+    print('Building %s with environment %s' % (test_project, project_env))
+    subprocess.check_call(['cibuildwheel', test_project], env=env)
+    wheels = glob('wheelhouse/*.whl')
+    print('%s built successfully. %i wheels built.' % (test_project, len(wheels)))
+
+    # check some wheels were actually built
+    assert len(wheels) >= 4
+
+    # clean up
+    shutil.rmtree('wheelhouse')
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("test_project_dir")
+    args = parser.parse_args()
+
+    project_path = os.path.abspath(args.test_project_dir)
+
+    # move cwd to the project root
+    os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+    if not os.path.exists(project_path):
+        print('No test project not found.', file=sys.stderr)
+        exit(2)
+
+    single_run(project_path)
+
+    print('Project built successfully.')

bin/run_tests.py

@@ -0,0 +1,30 @@
+#!/usr/bin/python
+
+from __future__ import print_function
+import os, sys, subprocess, shutil, json
+from glob import glob
+
+import run_test
+
+if __name__ == '__main__':
+    # move cwd to the project root
+    os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+    ### run the unit tests
+
+    subprocess.check_call(['python', '-m', 'pytest', 'unit_test'])
+
+    ### run the integration tests
+
+    test_projects = glob('test/??_*')
+
+    if len(test_projects) == 0:
+        print('No test projects found. Aborting.', file=sys.stderr)
+        exit(2)
+
+    print('Testing projects:', test_projects)
+
+    for project_path in test_projects:
+        run_test.single_run(project_path)
+
+    print('%d projects built successfully.' % len(test_projects))

cibuildwheel/__main__.py

@@ -1,8 +1,9 @@
 from __future__ import print_function
-import argparse, os, subprocess, sys, textwrap
+import argparse, os, subprocess, sys, textwrap, shlex
 
 import cibuildwheel
 import cibuildwheel.linux, cibuildwheel.windows, cibuildwheel.macos
+from cibuildwheel.environment import parse_environment, EnvironmentParseError
 from cibuildwheel.util import BuildSkipper
 
 def get_option_from_environment(option_name, platform=None):
@@ -72,6 +73,15 @@ def main():
     project_dir = args.project_dir
     before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform)
     skip_config = os.environ.get('CIBW_SKIP', '')
+    environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform) or ''
+
+    try:
+        environment = parse_environment(environment_config)
+    except (EnvironmentParseError, ValueError) as e:
+        print('cibuildwheel: Malformed environment option "%s"' % key_value, file=sys.stderr)
+        import traceback
+        traceback.print_exc(None, sys.stderr)
+        exit(2)
 
     skip = BuildSkipper(skip_config)
 
@@ -103,6 +113,7 @@ def main():
         test_requires=test_requires,
         before_build=before_build,
         skip=skip,
+        environment=environment,
     )
 
     print_preamble(platform, build_options)

cibuildwheel/bashlex_eval.py

@@ -0,0 +1,67 @@
+import subprocess, shlex
+from collections import namedtuple
+import bashlex
+
+NodeExecutionContext = namedtuple('NodeExecutionContext', ['environment', 'input'])
+
+def evaluate(value, environment):
+    if not value:
+        # empty string evaluates to empty string
+        # (but trips up bashlex)
+        return ''
+
+    command_node = bashlex.parsesingle(value)
+
+    if len(command_node.parts) != 1:
+        raise ValueError('"%s" has too many parts' % value)
+
+    value_word_node = command_node.parts[0]
+
+    return evaluate_node(
+        value_word_node, 
+        context=NodeExecutionContext(environment=environment, input=value)
+    )
+
+
+def evaluate_node(node, context):
+    if node.kind == 'word':
+        return evaluate_word_node(node, context=context)
+    elif node.kind == 'commandsubstitution':
+        return evaluate_command_node(node.command, context=context)
+    elif node.kind == 'parameter':
+        return evaluate_parameter_node(node, context=context)
+    else:
+        raise ValueError('Unsupported bash construct: "%s"' % node.word)
+
+
+def evaluate_word_node(node, context):
+    word_start = node.pos[0]
+    word_end = node.pos[1]
+    word_string = context.input[word_start:word_end]
+    letters = list(word_string)
+
+    for part in node.parts:
+        part_start = part.pos[0] - word_start
+        part_end = part.pos[1] - word_start
+
+        # Set all the characters in the part to None
+        for i in range(part_start, part_end):
+            letters[i] = None
+
+        letters[part_start] = evaluate_node(part, context=context)
+
+    # remove the None letters and concat
+    value = ''.join(l for l in letters if l is not None)
+
+    # apply bash-like quotes/whitespace treatment
+    return ' '.join(word.strip() for word in shlex.split(value))
+
+
+def evaluate_command_node(node, context):
+    words = [evaluate_node(part, context=context) for part in node.parts]
+    command = ' '.join(words)
+    return subprocess.check_output(shlex.split(command), env=context.environment)
+
+
+def evaluate_parameter_node(node, context):
+    return context.environment.get(node.value, '')

cibuildwheel/environment.py

@@ -0,0 +1,66 @@
+import bashlex
+from . import bashlex_eval
+
+
+class EnvironmentParseError(Exception):
+    pass
+
+
+def parse_environment(env_string):
+    env_items = split_env_items(env_string)
+    assignments = [EnvironmentAssignment(item) for item in env_items]
+    return ParsedEnvironment(assignments=assignments)
+
+
+def split_env_items(env_string):
+    '''Splits space-separated variable assignments into a list of individual assignments.
+
+    >>> split_env_items('VAR=abc')
+    ['VAR=abc']
+    >>> split_env_items('VAR="a string" THING=3')
+    ['VAR="a string"', 'THING=3']
+    >>> split_env_items('VAR="a string" THING=\\'single "quotes"\\'')
+    ['VAR="a string"', 'THING=\\'single "quotes"\\'']
+    >>> split_env_items('VAR="dont \\\\"forget\\\\" about backslashes"')
+    ['VAR="dont \\\\"forget\\\\" about backslashes"']
+    '''
+    return list(bashlex.split(env_string))
+
+
+class EnvironmentAssignment(object):
+    def __init__(self, assignment):
+        name, equals, value = assignment.partition('=')
+        if not equals:
+            raise EnvironmentParseError(assignment)
+        self.name = name
+        self.value = value
+
+    def evaluated_value(self, environment):
+        '''Returns the value of this assignment, as evaluated in the environment'''
+        return bashlex_eval.evaluate(self.value, environment=environment)
+
+    def as_shell_assignment(self):
+        return 'export %s=%s' % (self.name, self.value)
+
+    def __repr__(self):
+        return '%s=%s' % (self.name, self.value)
+
+
+class ParsedEnvironment(object):
+    def __init__(self, assignments):
+        self.assignments = assignments
+
+    def as_dictionary(self, prev_environment):
+        environment = prev_environment.copy()
+
+        for assignment in self.assignments:
+            value = assignment.evaluated_value(environment=environment)
+            environment[assignment.name] = value
+
+        return environment
+
+    def as_shell_commands(self):
+        return [a.as_shell_assignment() for a in self.assignments]
+
+    def __repr__(self):
+        return 'ParsedEnvironment(%r)' % [repr(a) for a in self.assignments]

cibuildwheel/linux.py

@@ -9,7 +9,7 @@
     from pipes import quote as shlex_quote
 
 
-def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip):
+def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip, environment):
     try:
         subprocess.check_call(['docker', '--version'])
     except:
@@ -52,6 +52,8 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
             set -o xtrace
             cd /project
 
+            {environment_exports}
+
             for PYBIN in {pybin_paths}; do
                 # Setup
                 rm -rf /tmp/built_wheel
@@ -106,6 +108,7 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
             before_build=shlex_quote(
                 prepare_command(before_build, python='python', pip='pip') if before_build else ''
             ),
+            environment_exports='\n'.join(environment.as_shell_commands()),
         )
 
         docker_process = subprocess.Popen([

cibuildwheel/macos.py

@@ -10,7 +10,7 @@
 from .util import prepare_command
 
 
-def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip):
+def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip, environment):
     PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url'])
     python_configurations = [
         PythonConfiguration(version='2.7', identifier='cp27-macosx_10_6_intel', url='https://www.python.org/ftp/python/2.7.13/python-2.7.13-macosx10.6.pkg'),
@@ -39,6 +39,7 @@ def shell(args, env=None, cwd=None):
             '/Library/Frameworks/Python.framework/Versions/%s/bin' % config.version,
             env['PATH'],
         ])
+        env = environment.as_dictionary(prev_environment=env)
 
         python = 'python3' if config.version[0] == '3' else 'python2'
         pip = 'pip3' if config.version[0] == '3' else 'pip2'

cibuildwheel/windows.py

@@ -10,7 +10,7 @@
 from .util import prepare_command
 
 
-def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip):
+def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip, environment):
     # Python under AppVeyor/Windows seems to be buffering by default, giving problems interleaving subprocess call output with unflushed calls to 'print'
     sys.stdout.flush()
     sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
@@ -64,6 +64,7 @@ def shell(args, env=None, cwd=None):
             os.path.join(config.path, 'Scripts'),
             env['PATH']
         ])
+        env = environment.as_dictionary(prev_environment=env)
 
         # for the logs - check we're running the right version of python
         shell(['python', '--version'], env=env)

requirements-dev.txt

@@ -0,0 +1,2 @@
+-e .
+pytest

setup.py

@@ -9,6 +9,7 @@
 setup(
     name='cibuildwheel',
     version='0.4.1',
+    install_requires=['bashlex'],
     description="Build Python wheels on CI with minimal configuration.",
     long_description='For readme please see http://github.com/joerick/cibuildwheel',
     author="Joe Rickerby",

test/05_environment/environment.json

@@ -0,0 +1,4 @@
+{
+    "CIBW_ENVIRONMENT": "CIBW_TEST_VAR=\"a b c\" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3=\"$(echo 'test string 3')\" PATH=$PATH:/opt/cibw_test_path",
+    "CIBW_ENVIRONMENT_WINDOWS": "CIBW_TEST_VAR=\"a b c\" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3=\"$(echo 'test string 3')\" PATH=\"$PATH;/opt/cibw_test_path\""
+}