Merge pull request #342 from Czaki/setup_data

Add BEFORE_ALL options

cibuildwheel/__main__.py

@@ -128,6 +128,7 @@ def main() -> None:
 
     build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '')
     environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='')
+    before_all = get_option_from_environment('CIBW_BEFORE_ALL', platform=platform, default='')
     before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform)
     repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default)
     dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned')
@@ -212,6 +213,7 @@ def main() -> None:
         test_extras=test_extras,
         before_test=before_test,
         before_build=before_build,
+        before_all=before_all,
         build_verbosity=build_verbosity,
         build_selector=build_selector,
         repair_command=repair_command,

cibuildwheel/environment.py

@@ -1,6 +1,6 @@
 import bashlex  # type: ignore
 
-from typing import Dict, List
+from typing import Dict, List, Mapping
 
 from . import bashlex_eval
 
@@ -61,8 +61,8 @@ class ParsedEnvironment:
     def __init__(self, assignments: List[EnvironmentAssignment]):
         self.assignments = assignments
 
-    def as_dictionary(self, prev_environment: Dict[str, str]) -> Dict[str, str]:
-        environment = prev_environment.copy()
+    def as_dictionary(self, prev_environment: Mapping[str, str]) -> Dict[str, str]:
+        environment = dict(**prev_environment)
 
         for assignment in self.assignments:
             value = assignment.evaluated_value(environment=environment)

cibuildwheel/linux.py

@@ -133,6 +133,30 @@ def build(options: BuildOptions) -> None:
 
             call(['docker', 'start', container_name])
 
+            if options.before_all:
+                call(
+                    ['docker', 'exec', '-i', container_name] + shell_cmd,
+                    universal_newlines=True,
+                    input='''
+                        PS4='    + '
+
+                        set -o errexit
+                        set -o xtrace
+
+                        # add a modern Python interpreter to PATH so it can be used by BEFORE_ALL
+                        # commands
+                        export PATH=/opt/python/cp38-cp38:$PATH
+
+                        {environment_exports}
+
+                        sh -c {before_all}
+
+                    '''.format(
+                        environment_exports='\n'.join(options.environment.as_shell_commands()),
+                        before_all=shlex.quote(prepare_command(options.before_all, project='/project', package=container_package_dir))
+                    )
+                )
+
             for config in platform_configs:
                 if options.dependency_constraints:
                     constraints_file = options.dependency_constraints.get_for_python_version(config.version)

cibuildwheel/macos.py

@@ -176,6 +176,11 @@ def build(options: BuildOptions) -> None:
     built_wheel_dir = temp_dir / 'built_wheel'
     repaired_wheel_dir = temp_dir / 'repaired_wheel'
 
+    if options.before_all:
+        env = options.environment.as_dictionary(prev_environment=os.environ)
+        before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir)
+        call([before_all_prepared], shell=True, env=env)
+
     python_configurations = get_python_configurations(options.build_selector)
 
     for config in python_configurations:

cibuildwheel/util.py

@@ -112,6 +112,7 @@ class BuildOptions(NamedTuple):
     output_dir: Path
     build_selector: BuildSelector
     environment: ParsedEnvironment
+    before_all: str
     before_build: Optional[str]
     repair_command: str
     manylinux_images: Optional[Dict[str, str]]

cibuildwheel/windows.py

@@ -162,6 +162,11 @@ def build(options: BuildOptions) -> None:
     nuget = Path('C:\\cibw\\nuget.exe')
     download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
 
+    if options.before_all:
+        env = options.environment.as_dictionary(prev_environment=os.environ)
+        before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir)
+        shell([before_all_prepared], env=env)
+
     python_configurations = get_python_configurations(options.build_selector)
     for config in python_configurations:
         dependency_constraint_flags = []

docs/options.md

@@ -190,6 +190,28 @@ CIBW_ENVIRONMENT: "BUILD_TIME=$(date) SAMPLE_TEXT=\"sample text\""
 !!! note
     `cibuildwheel` always defines the environment variable `CIBUILDWHEEL=1`. This can be useful for [building wheels with optional extensions](faq.md#building-packages-with-optional-c-extensions).
 
+### `CIBW_BEFORE_ALL` {: #before-all}
+> Execute a shell command on the build system before any wheels are built.
+
+Shell command to prepare a common part of the project (e.g. build or install libraries which does not depend on the specific version of Python).
+
+This option is very useful for the Linux build, where builds take place in isolated Docker containers managed by cibuildwheel. This command will run inside the container before the wheel builds start. Note, if you're building both x86_64 and i686 wheels (the default), your build uses two different Docker images. In that case, this command will execute twice - once per build container.
+
+The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`.
+
+On Windows and macOS, the version of Python available inside `CIBW_BEFORE_ALL` is whatever is available on the host machine. On Linux, a modern Python version is available on PATH.
+
+Platform-specific variants also available:<br/>
+ `CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX`
+
+#### Examples
+```yaml
+# build third party library
+CIBW_BEFORE_ALL: make -C third_party_lib
+
+# install system library
+CIBW_BEFORE_ALL_LINUX: yum install -y libffi-dev
+```
 
 ### `CIBW_BEFORE_BUILD` {: #before-build}
 > Execute a shell command preparing each wheel's build

test/test_before_all.py

@@ -0,0 +1,38 @@
+import os
+import textwrap
+
+from . import utils
+from . import test_projects
+
+project_with_before_build_asserts = test_projects.new_c_project(
+    setup_py_add=textwrap.dedent(r'''
+        # assert that the Python version as written to text_info.txt in the CIBW_BEFORE_ALL step
+        # is the same one as is currently running.
+        with open("text_info.txt") as f:
+            stored_text = f.read()
+
+        print("## stored text: " + stored_text)
+        assert stored_text == "sample text 123"
+    ''')
+)
+
+
+def test(tmp_path):
+    project_dir = tmp_path / 'project'
+    project_with_before_build_asserts.generate(project_dir)
+
+    with open(os.path.join(project_dir, "text_info.txt"), mode='w') as ff:
+        print("dummy text", file=ff)
+
+    # build the wheels
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
+        # write python version information to a temporary file, this is
+        # checked in setup.py
+        'CIBW_BEFORE_ALL': '''python -c "import os;open('{project}/text_info.txt', 'w').write('sample text '+os.environ.get('TEST_VAL', ''))"''',
+        'CIBW_ENVIRONMENT': "TEST_VAL='123'"
+    })
+
+    # also check that we got the right wheels
+    os.remove(os.path.join(project_dir, "text_info.txt"))
+    expected_wheels = utils.expected_wheels('spam', '0.1.0')
+    assert set(actual_wheels) == set(expected_wheels)

unit_test/main_tests/main_options_test.py

@@ -233,3 +233,21 @@ def test_build_selector_migrations(intercepted_build_args, monkeypatch, option_n
         assert intercepted_build_selector.build_patterns == build_selector_patterns
     else:
         assert intercepted_build_selector.skip_patterns == build_selector_patterns
+
+
+@pytest.mark.parametrize('before_all', ["", None, 'test text'])
+@pytest.mark.parametrize('platform_specific', [False, True])
+def test_before_all(before_all, platform_specific, platform, intercepted_build_args, monkeypatch):
+    if before_all is not None:
+        if platform_specific:
+            monkeypatch.setenv('CIBW_BEFORE_ALL_' + platform.upper(), before_all)
+            monkeypatch.setenv('CIBW_BEFORE_ALL', 'overwritten')
+        else:
+            monkeypatch.setenv('CIBW_BEFORE_ALL', before_all)
+
+    main()
+
+    if before_all is None:
+        before_all = ""
+
+    assert intercepted_build_args.args[0].before_all == before_all