Merge branch 'master' into henryiii/pep517

.circleci/prepare.sh

@@ -1,6 +1,8 @@
+set -o errexit
+set -o xtrace
+
 $PYTHON --version
 $PYTHON -m pip --version
-$PYTHON -m pip install -q --user --ignore-installed --upgrade virtualenv
 $PYTHON -m virtualenv -p $PYTHON venv
 venv/bin/python -m pip install -r requirements-dev.txt
 venv/bin/python -m pip freeze

.github/auto-merge.yml

@@ -0,0 +1,7 @@
+# config file for https://github.com/bobvanderlinden/probot-auto-merge
+
+# no approvals required, just the label is enough
+minApprovals: {}
+
+requiredLabels:
+- merge when green

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+repos:
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.1
+    hooks:
+    - id: flake8
+      pass_filenames: true
+      # this seems to need to be here in addition to setup.cfg
+      exclude: cibuildwheel/resources/
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    - id: check-yaml
+    - id: check-merge-conflict
+    - id: end-of-file-fixer
+      exclude: cibuildwheel/resources/pinned_docker_images.cfg
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: ''  # Use the sha / tag you want to point at
+    hooks:
+    - id: mypy
+      files: cibuildwheel|test
+      exclude: cibuildwheel/resources

README.md

@@ -102,17 +102,20 @@ Options
 |   | Option | Description |
 |---|--------|-------------|
 | **Build selection** | [`CIBW_PLATFORM`](https://cibuildwheel.readthedocs.io/en/stable/options/#platform)  | Override the auto-detected target platform |
-|   | [`CIBW_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)  [`CIBW_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)  | Choose the Python versions to build |
-| **Build environment** | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment)  | Set environment variables needed during the build |
+|   | [`CIBW_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)  <br> [`CIBW_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)  | Choose the Python versions to build |
+| **Build customization** | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment)  | Set environment variables needed during the build |
+|   | [`CIBW_BEFORE_ALL`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-all)  | Execute a shell command on the build system before any wheels are built. |
 |   | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build)  | Execute a shell command preparing each wheel's build |
 |   | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command)  | Execute a shell command to repair each (non-pure Python) built wheel |
-|   | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)  [`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)  [`CIBW_MANYLINUX_PYPY_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)  | Specify alternative manylinux docker images |
+|   | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)  <br> [`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)  <br> [`CIBW_MANYLINUX_PYPY_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)  <br> [`CIBW_MANYLINUX_AARCH64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)  <br> [`CIBW_MANYLINUX_PPC64LE_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)  <br> [`CIBW_MANYLINUX_S390X_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)  | Specify alternative manylinux docker images |
+|   | [`CIBW_DEPENDENCY_VERSIONS`](https://cibuildwheel.readthedocs.io/en/stable/options/#dependency-versions)  | Specify how cibuildwheel controls the versions of the tools it uses |
 | **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command)  | Execute a shell command to test each built wheel |
-|   | [`CIBW_BEFORE_TEST`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-test)  | Execute shell command to prepare test environment |
+|   | [`CIBW_BEFORE_TEST`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-test)  | Execute a shell command before testing each wheel |
 |   | [`CIBW_TEST_REQUIRES`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-requires)  | Install Python dependencies before running the tests |
 |   | [`CIBW_TEST_EXTRAS`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras)  | Install your wheel for testing using extras_require |
 | **Other** | [`CIBW_BUILD_VERBOSITY`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-verbosity)  | Increase/decrease the output of pip wheel |
 
+
 Working examples
 ----------------
 

azure-pipelines.yml

@@ -42,4 +42,3 @@ jobs:
     - bash: |
         python -m pip install -r requirements-dev.txt pytest-azurepipelines
         python ./bin/run_tests.py
-

cibuildwheel/__main__.py

@@ -128,6 +128,7 @@ def main() -> None:
 
     build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '')
     environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='')
+    before_all = get_option_from_environment('CIBW_BEFORE_ALL', platform=platform, default='')
     before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform)
     repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default)
     dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned')
@@ -212,6 +213,7 @@ def main() -> None:
         test_extras=test_extras,
         before_test=before_test,
         before_build=before_build,
+        before_all=before_all,
         build_verbosity=build_verbosity,
         build_selector=build_selector,
         repair_command=repair_command,

cibuildwheel/environment.py

@@ -1,6 +1,6 @@
 import bashlex  # type: ignore
 
-from typing import Dict, List
+from typing import Dict, List, Mapping
 
 from . import bashlex_eval
 
@@ -61,8 +61,8 @@ class ParsedEnvironment:
     def __init__(self, assignments: List[EnvironmentAssignment]):
         self.assignments = assignments
 
-    def as_dictionary(self, prev_environment: Dict[str, str]) -> Dict[str, str]:
-        environment = prev_environment.copy()
+    def as_dictionary(self, prev_environment: Mapping[str, str]) -> Dict[str, str]:
+        environment = dict(**prev_environment)
 
         for assignment in self.assignments:
             value = assignment.evaluated_value(environment=environment)

cibuildwheel/linux.py

@@ -133,6 +133,30 @@ def build(options: BuildOptions) -> None:
 
             call(['docker', 'start', container_name])
 
+            if options.before_all:
+                call(
+                    ['docker', 'exec', '-i', container_name] + shell_cmd,
+                    universal_newlines=True,
+                    input='''
+                        PS4='    + '
+
+                        set -o errexit
+                        set -o xtrace
+
+                        # add a modern Python interpreter to PATH so it can be used by BEFORE_ALL
+                        # commands
+                        export PATH=/opt/python/cp38-cp38:$PATH
+
+                        {environment_exports}
+
+                        sh -c {before_all}
+
+                    '''.format(
+                        environment_exports='\n'.join(options.environment.as_shell_commands()),
+                        before_all=shlex.quote(prepare_command(options.before_all, project='/project', package=container_package_dir))
+                    )
+                )
+
             for config in platform_configs:
                 if options.dependency_constraints:
                     constraints_file = options.dependency_constraints.get_for_python_version(config.version)

cibuildwheel/macos.py

@@ -176,6 +176,11 @@ def build(options: BuildOptions) -> None:
     built_wheel_dir = temp_dir / 'built_wheel'
     repaired_wheel_dir = temp_dir / 'repaired_wheel'
 
+    if options.before_all:
+        env = options.environment.as_dictionary(prev_environment=os.environ)
+        before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir)
+        call([before_all_prepared], shell=True, env=env)
+
     python_configurations = get_python_configurations(options.build_selector)
 
     for config in python_configurations:

cibuildwheel/util.py

@@ -112,6 +112,7 @@ class BuildOptions(NamedTuple):
     output_dir: Path
     build_selector: BuildSelector
     environment: ParsedEnvironment
+    before_all: str
     before_build: Optional[str]
     repair_command: str
     manylinux_images: Optional[Dict[str, str]]

cibuildwheel/windows.py

@@ -188,6 +188,11 @@ def build(options: BuildOptions) -> None:
     nuget = Path('C:\\cibw\\nuget.exe')
     download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
 
+    if options.before_all:
+        env = options.environment.as_dictionary(prev_environment=os.environ)
+        before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir)
+        shell([before_all_prepared], env=env)
+
     python_configurations = get_python_configurations(options.build_selector)
     for config in python_configurations:
         dependency_constraint_flags = []

docs/extra.js

@@ -2,4 +2,3 @@
 $('.rst-content pre')
     .prev('blockquote')
     .addClass('code-block-filename');
-

docs/options.md

@@ -190,6 +190,28 @@ CIBW_ENVIRONMENT: "BUILD_TIME=$(date) SAMPLE_TEXT=\"sample text\""
 !!! note
     `cibuildwheel` always defines the environment variable `CIBUILDWHEEL=1`. This can be useful for [building wheels with optional extensions](faq.md#building-packages-with-optional-c-extensions).
 
+### `CIBW_BEFORE_ALL` {: #before-all}
+> Execute a shell command on the build system before any wheels are built.
+
+Shell command to prepare a common part of the project (e.g. build or install libraries which does not depend on the specific version of Python).
+
+This option is very useful for the Linux build, where builds take place in isolated Docker containers managed by cibuildwheel. This command will run inside the container before the wheel builds start. Note, if you're building both x86_64 and i686 wheels (the default), your build uses two different Docker images. In that case, this command will execute twice - once per build container.
+
+The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`.
+
+On Windows and macOS, the version of Python available inside `CIBW_BEFORE_ALL` is whatever is available on the host machine. On Linux, a modern Python version is available on PATH.
+
+Platform-specific variants also available:<br/>
+ `CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX`
+
+#### Examples
+```yaml
+# build third party library
+CIBW_BEFORE_ALL: make -C third_party_lib
+
+# install system library
+CIBW_BEFORE_ALL_LINUX: yum install -y libffi-dev
+```
 
 ### `CIBW_BEFORE_BUILD` {: #before-build}
 > Execute a shell command preparing each wheel's build
@@ -603,7 +625,7 @@ optional arguments:
         var url = 'https://cibuildwheel.readthedocs.io/en/stable/options/#'+option.id;
         var namesMarkdown = $.map(optionNames, function(n) {
           return '[`'+n+'`]('+url+') '
-        }).join(' ')
+        }).join(' <br> ')
 
         markdown += '| '+namesMarkdown+' '
         markdown += '| '+option.description.trim()+' '

test/test_before_all.py

@@ -0,0 +1,38 @@
+import os
+import textwrap
+
+from . import utils
+from . import test_projects
+
+project_with_before_build_asserts = test_projects.new_c_project(
+    setup_py_add=textwrap.dedent(r'''
+        # assert that the Python version as written to text_info.txt in the CIBW_BEFORE_ALL step
+        # is the same one as is currently running.
+        with open("text_info.txt") as f:
+            stored_text = f.read()
+
+        print("## stored text: " + stored_text)
+        assert stored_text == "sample text 123"
+    ''')
+)
+
+
+def test(tmp_path):
+    project_dir = tmp_path / 'project'
+    project_with_before_build_asserts.generate(project_dir)
+
+    with open(os.path.join(project_dir, "text_info.txt"), mode='w') as ff:
+        print("dummy text", file=ff)
+
+    # build the wheels
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
+        # write python version information to a temporary file, this is
+        # checked in setup.py
+        'CIBW_BEFORE_ALL': '''python -c "import os;open('{project}/text_info.txt', 'w').write('sample text '+os.environ.get('TEST_VAL', ''))"''',
+        'CIBW_ENVIRONMENT': "TEST_VAL='123'"
+    })
+
+    # also check that we got the right wheels
+    os.remove(os.path.join(project_dir, "text_info.txt"))
+    expected_wheels = utils.expected_wheels('spam', '0.1.0')
+    assert set(actual_wheels) == set(expected_wheels)

unit_test/main_tests/main_options_test.py

@@ -233,3 +233,21 @@ def test_build_selector_migrations(intercepted_build_args, monkeypatch, option_n
         assert intercepted_build_selector.build_patterns == build_selector_patterns
     else:
         assert intercepted_build_selector.skip_patterns == build_selector_patterns
+
+
+@pytest.mark.parametrize('before_all', ["", None, 'test text'])
+@pytest.mark.parametrize('platform_specific', [False, True])
+def test_before_all(before_all, platform_specific, platform, intercepted_build_args, monkeypatch):
+    if before_all is not None:
+        if platform_specific:
+            monkeypatch.setenv('CIBW_BEFORE_ALL_' + platform.upper(), before_all)
+            monkeypatch.setenv('CIBW_BEFORE_ALL', 'overwritten')
+        else:
+            monkeypatch.setenv('CIBW_BEFORE_ALL', before_all)
+
+    main()
+
+    if before_all is None:
+        before_all = ""
+
+    assert intercepted_build_args.args[0].before_all == before_all