diff --git a/.circleci/prepare.sh b/.circleci/prepare.sh
index dfd3aeef..7256c8c3 100644
--- a/.circleci/prepare.sh
+++ b/.circleci/prepare.sh
@@ -1,6 +1,8 @@
+set -o errexit
+set -o xtrace
+
$PYTHON --version
$PYTHON -m pip --version
-$PYTHON -m pip install -q --user --ignore-installed --upgrade virtualenv
$PYTHON -m virtualenv -p $PYTHON venv
venv/bin/python -m pip install -r requirements-dev.txt
venv/bin/python -m pip freeze
diff --git a/.github/auto-merge.yml b/.github/auto-merge.yml
new file mode 100644
index 00000000..a461ed1d
--- /dev/null
+++ b/.github/auto-merge.yml
@@ -0,0 +1,7 @@
+# config file for https://github.com/bobvanderlinden/probot-auto-merge
+
+# no approvals required, just the label is enough
+minApprovals: {}
+
+requiredLabels:
+- merge when green
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 00000000..15c843f9
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,21 @@
+repos:
+- repo: https://gitlab.com/pycqa/flake8
+ rev: 3.8.1
+ hooks:
+ - id: flake8
+ pass_filenames: true
+ # this seems to need to be here in addition to setup.cfg
+ exclude: cibuildwheel/resources/
+- repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v2.3.0
+ hooks:
+ - id: check-yaml
+ - id: check-merge-conflict
+ - id: end-of-file-fixer
+ exclude: cibuildwheel/resources/pinned_docker_images.cfg
+- repo: https://github.com/pre-commit/mirrors-mypy
+ rev: '' # Use the sha / tag you want to point at
+ hooks:
+ - id: mypy
+ files: cibuildwheel|test
+ exclude: cibuildwheel/resources
diff --git a/README.md b/README.md
index d3805547..3412159e 100644
--- a/README.md
+++ b/README.md
@@ -102,17 +102,20 @@ Options
| | Option | Description |
|---|--------|-------------|
| **Build selection** | [`CIBW_PLATFORM`](https://cibuildwheel.readthedocs.io/en/stable/options/#platform) | Override the auto-detected target platform |
-| | [`CIBW_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip) [`CIBW_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip) | Choose the Python versions to build |
-| **Build environment** | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment) | Set environment variables needed during the build |
+| | [`CIBW_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)
[`CIBW_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip) | Choose the Python versions to build |
+| **Build customization** | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment) | Set environment variables needed during the build |
+| | [`CIBW_BEFORE_ALL`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-all) | Execute a shell command on the build system before any wheels are built. |
| | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build |
| | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each (non-pure Python) built wheel |
-| | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_PYPY_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) | Specify alternative manylinux docker images |
+| | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)
[`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)
[`CIBW_MANYLINUX_PYPY_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)
[`CIBW_MANYLINUX_AARCH64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)
[`CIBW_MANYLINUX_PPC64LE_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)
[`CIBW_MANYLINUX_S390X_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) | Specify alternative manylinux docker images |
+| | [`CIBW_DEPENDENCY_VERSIONS`](https://cibuildwheel.readthedocs.io/en/stable/options/#dependency-versions) | Specify how cibuildwheel controls the versions of the tools it uses |
| **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command) | Execute a shell command to test each built wheel |
-| | [`CIBW_BEFORE_TEST`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-test) | Execute shell command to prepare test environment |
+| | [`CIBW_BEFORE_TEST`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-test) | Execute a shell command before testing each wheel |
| | [`CIBW_TEST_REQUIRES`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-requires) | Install Python dependencies before running the tests |
| | [`CIBW_TEST_EXTRAS`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras) | Install your wheel for testing using extras_require |
| **Other** | [`CIBW_BUILD_VERBOSITY`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-verbosity) | Increase/decrease the output of pip wheel |
+
Working examples
----------------
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 4fb53b11..3eaed5f4 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -42,4 +42,3 @@ jobs:
- bash: |
python -m pip install -r requirements-dev.txt pytest-azurepipelines
python ./bin/run_tests.py
-
diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py
index 137bc19e..4e25befa 100644
--- a/cibuildwheel/__main__.py
+++ b/cibuildwheel/__main__.py
@@ -128,6 +128,7 @@ def main() -> None:
build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '')
environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='')
+ before_all = get_option_from_environment('CIBW_BEFORE_ALL', platform=platform, default='')
before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform)
repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default)
dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned')
@@ -212,6 +213,7 @@ def main() -> None:
test_extras=test_extras,
before_test=before_test,
before_build=before_build,
+ before_all=before_all,
build_verbosity=build_verbosity,
build_selector=build_selector,
repair_command=repair_command,
diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py
index 086a253c..9f7ea2c4 100644
--- a/cibuildwheel/environment.py
+++ b/cibuildwheel/environment.py
@@ -1,6 +1,6 @@
import bashlex # type: ignore
-from typing import Dict, List
+from typing import Dict, List, Mapping
from . import bashlex_eval
@@ -61,8 +61,8 @@ class ParsedEnvironment:
def __init__(self, assignments: List[EnvironmentAssignment]):
self.assignments = assignments
- def as_dictionary(self, prev_environment: Dict[str, str]) -> Dict[str, str]:
- environment = prev_environment.copy()
+ def as_dictionary(self, prev_environment: Mapping[str, str]) -> Dict[str, str]:
+ environment = dict(**prev_environment)
for assignment in self.assignments:
value = assignment.evaluated_value(environment=environment)
diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py
index 99e779c4..6e42f47f 100644
--- a/cibuildwheel/linux.py
+++ b/cibuildwheel/linux.py
@@ -133,6 +133,30 @@ def build(options: BuildOptions) -> None:
call(['docker', 'start', container_name])
+ if options.before_all:
+ call(
+ ['docker', 'exec', '-i', container_name] + shell_cmd,
+ universal_newlines=True,
+ input='''
+ PS4=' + '
+
+ set -o errexit
+ set -o xtrace
+
+ # add a modern Python interpreter to PATH so it can be used by BEFORE_ALL
+ # commands
+ export PATH=/opt/python/cp38-cp38:$PATH
+
+ {environment_exports}
+
+ sh -c {before_all}
+
+ '''.format(
+ environment_exports='\n'.join(options.environment.as_shell_commands()),
+ before_all=shlex.quote(prepare_command(options.before_all, project='/project', package=container_package_dir))
+ )
+ )
+
for config in platform_configs:
if options.dependency_constraints:
constraints_file = options.dependency_constraints.get_for_python_version(config.version)
diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py
index 2fd28d20..d9d892b9 100644
--- a/cibuildwheel/macos.py
+++ b/cibuildwheel/macos.py
@@ -176,6 +176,11 @@ def build(options: BuildOptions) -> None:
built_wheel_dir = temp_dir / 'built_wheel'
repaired_wheel_dir = temp_dir / 'repaired_wheel'
+ if options.before_all:
+ env = options.environment.as_dictionary(prev_environment=os.environ)
+ before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir)
+ call([before_all_prepared], shell=True, env=env)
+
python_configurations = get_python_configurations(options.build_selector)
for config in python_configurations:
diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py
index feef6185..012afa1a 100644
--- a/cibuildwheel/util.py
+++ b/cibuildwheel/util.py
@@ -112,6 +112,7 @@ class BuildOptions(NamedTuple):
output_dir: Path
build_selector: BuildSelector
environment: ParsedEnvironment
+ before_all: str
before_build: Optional[str]
repair_command: str
manylinux_images: Optional[Dict[str, str]]
diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py
index f8a50015..31cbccb4 100644
--- a/cibuildwheel/windows.py
+++ b/cibuildwheel/windows.py
@@ -188,6 +188,11 @@ def build(options: BuildOptions) -> None:
nuget = Path('C:\\cibw\\nuget.exe')
download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
+ if options.before_all:
+ env = options.environment.as_dictionary(prev_environment=os.environ)
+ before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir)
+ shell([before_all_prepared], env=env)
+
python_configurations = get_python_configurations(options.build_selector)
for config in python_configurations:
dependency_constraint_flags = []
diff --git a/docs/extra.js b/docs/extra.js
index d32c3cc0..1965a84e 100644
--- a/docs/extra.js
+++ b/docs/extra.js
@@ -2,4 +2,3 @@
$('.rst-content pre')
.prev('blockquote')
.addClass('code-block-filename');
-
diff --git a/docs/options.md b/docs/options.md
index 18d5417b..856f0bad 100644
--- a/docs/options.md
+++ b/docs/options.md
@@ -190,6 +190,28 @@ CIBW_ENVIRONMENT: "BUILD_TIME=$(date) SAMPLE_TEXT=\"sample text\""
!!! note
`cibuildwheel` always defines the environment variable `CIBUILDWHEEL=1`. This can be useful for [building wheels with optional extensions](faq.md#building-packages-with-optional-c-extensions).
+### `CIBW_BEFORE_ALL` {: #before-all}
+> Execute a shell command on the build system before any wheels are built.
+
+Shell command to prepare a common part of the project (e.g. build or install libraries which does not depend on the specific version of Python).
+
+This option is very useful for the Linux build, where builds take place in isolated Docker containers managed by cibuildwheel. This command will run inside the container before the wheel builds start. Note, if you're building both x86_64 and i686 wheels (the default), your build uses two different Docker images. In that case, this command will execute twice - once per build container.
+
+The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`.
+
+On Windows and macOS, the version of Python available inside `CIBW_BEFORE_ALL` is whatever is available on the host machine. On Linux, a modern Python version is available on PATH.
+
+Platform-specific variants also available:
+ `CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX`
+
+#### Examples
+```yaml
+# build third party library
+CIBW_BEFORE_ALL: make -C third_party_lib
+
+# install system library
+CIBW_BEFORE_ALL_LINUX: yum install -y libffi-dev
+```
### `CIBW_BEFORE_BUILD` {: #before-build}
> Execute a shell command preparing each wheel's build
@@ -603,7 +625,7 @@ optional arguments:
var url = 'https://cibuildwheel.readthedocs.io/en/stable/options/#'+option.id;
var namesMarkdown = $.map(optionNames, function(n) {
return '[`'+n+'`]('+url+') '
- }).join(' ')
+ }).join('
')
markdown += '| '+namesMarkdown+' '
markdown += '| '+option.description.trim()+' '
diff --git a/test/test_before_all.py b/test/test_before_all.py
new file mode 100644
index 00000000..177ec555
--- /dev/null
+++ b/test/test_before_all.py
@@ -0,0 +1,38 @@
+import os
+import textwrap
+
+from . import utils
+from . import test_projects
+
+project_with_before_build_asserts = test_projects.new_c_project(
+ setup_py_add=textwrap.dedent(r'''
+ # assert that the Python version as written to text_info.txt in the CIBW_BEFORE_ALL step
+ # is the same one as is currently running.
+ with open("text_info.txt") as f:
+ stored_text = f.read()
+
+ print("## stored text: " + stored_text)
+ assert stored_text == "sample text 123"
+ ''')
+)
+
+
+def test(tmp_path):
+ project_dir = tmp_path / 'project'
+ project_with_before_build_asserts.generate(project_dir)
+
+ with open(os.path.join(project_dir, "text_info.txt"), mode='w') as ff:
+ print("dummy text", file=ff)
+
+ # build the wheels
+ actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
+ # write python version information to a temporary file, this is
+ # checked in setup.py
+ 'CIBW_BEFORE_ALL': '''python -c "import os;open('{project}/text_info.txt', 'w').write('sample text '+os.environ.get('TEST_VAL', ''))"''',
+ 'CIBW_ENVIRONMENT': "TEST_VAL='123'"
+ })
+
+ # also check that we got the right wheels
+ os.remove(os.path.join(project_dir, "text_info.txt"))
+ expected_wheels = utils.expected_wheels('spam', '0.1.0')
+ assert set(actual_wheels) == set(expected_wheels)
diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py
index 14252e67..c7e6bfdd 100644
--- a/unit_test/main_tests/main_options_test.py
+++ b/unit_test/main_tests/main_options_test.py
@@ -233,3 +233,21 @@ def test_build_selector_migrations(intercepted_build_args, monkeypatch, option_n
assert intercepted_build_selector.build_patterns == build_selector_patterns
else:
assert intercepted_build_selector.skip_patterns == build_selector_patterns
+
+
+@pytest.mark.parametrize('before_all', ["", None, 'test text'])
+@pytest.mark.parametrize('platform_specific', [False, True])
+def test_before_all(before_all, platform_specific, platform, intercepted_build_args, monkeypatch):
+ if before_all is not None:
+ if platform_specific:
+ monkeypatch.setenv('CIBW_BEFORE_ALL_' + platform.upper(), before_all)
+ monkeypatch.setenv('CIBW_BEFORE_ALL', 'overwritten')
+ else:
+ monkeypatch.setenv('CIBW_BEFORE_ALL', before_all)
+
+ main()
+
+ if before_all is None:
+ before_all = ""
+
+ assert intercepted_build_args.args[0].before_all == before_all