From 6501470357e6ccec3090f5eaafe0f834e42eab69 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Mon, 11 May 2020 17:09:54 +0200 Subject: [PATCH 01/15] Add BEFORE_ALL options --- cibuildwheel/__main__.py | 2 ++ cibuildwheel/linux.py | 4 +++ cibuildwheel/macos.py | 4 +++ cibuildwheel/util.py | 3 +- cibuildwheel/windows.py | 4 +++ docs/options.md | 19 ++++++++++ test/14_before_all/cibuildwheel_test.py | 23 ++++++++++++ test/14_before_all/setup.py | 23 ++++++++++++ test/14_before_all/spam.c | 48 +++++++++++++++++++++++++ 9 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 test/14_before_all/cibuildwheel_test.py create mode 100644 test/14_before_all/setup.py create mode 100644 test/14_before_all/spam.c diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 137bc19e..bce3f754 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -128,6 +128,7 @@ def main() -> None: build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '') environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='') + before_all = get_option_from_environment('CIBW_BEFORE_ALL', platform=platform) before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default) dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned') @@ -212,6 +213,7 @@ def main() -> None: test_extras=test_extras, before_test=before_test, before_build=before_build, + before_all=before_all, build_verbosity=build_verbosity, build_selector=build_selector, repair_command=repair_command, diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 99e779c4..8bbda24d 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -133,6 +133,10 @@ def build(options: BuildOptions) -> None: call(['docker', 'start', container_name]) + if options.before_all: + before_all_prepared = prepare_command(options.before_all, project='/project', package=options.package_dir) + call(['docker', 'exec', '-i', container_name] + shell_cmd, universal_newlines=True, input=before_all_prepared) + for config in platform_configs: if options.dependency_constraints: constraints_file = options.dependency_constraints.get_for_python_version(config.version) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 2fd28d20..3e67b534 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -176,6 +176,10 @@ def build(options: BuildOptions) -> None: built_wheel_dir = temp_dir / 'built_wheel' repaired_wheel_dir = temp_dir / 'repaired_wheel' + if options.before_all: + before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) + call([before_all_prepared], shell=True) + python_configurations = get_python_configurations(options.build_selector) for config in python_configurations: diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index feef6185..f7201462 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -112,6 +112,7 @@ class BuildOptions(NamedTuple): output_dir: Path build_selector: BuildSelector environment: ParsedEnvironment + before_all: str before_build: Optional[str] repair_command: str manylinux_images: Optional[Dict[str, str]] @@ -120,7 +121,7 @@ class BuildOptions(NamedTuple): before_test: Optional[str] test_requires: List[str] test_extras: str - build_verbosity: int +x build_verbosity: int resources_dir = Path(__file__).resolve().parent / 'resources' diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index e81fe301..73628c7f 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -162,6 +162,10 @@ def build(options: BuildOptions) -> None: nuget = Path('C:\\cibw\\nuget.exe') download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) + if options.before_all: + before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) + shell([before_all_prepared]) + python_configurations = get_python_configurations(options.build_selector) for config in python_configurations: dependency_constraint_flags = [] diff --git a/docs/options.md b/docs/options.md index 18d5417b..0d61060d 100644 --- a/docs/options.md +++ b/docs/options.md @@ -190,6 +190,25 @@ CIBW_ENVIRONMENT: "BUILD_TIME=$(date) SAMPLE_TEXT=\"sample text\"" !!! note `cibuildwheel` always defines the environment variable `CIBUILDWHEEL=1`. This can be useful for [building wheels with optional extensions](faq.md#building-packages-with-optional-c-extensions). +### `CIBW_BEFORE_ALL` {: #before-all} +> Execute a shell command preparing common part for each wheel. + +Shell command to prepare common part of project (ex. build libraries which does not depend on python). +This option is added mainly for linux build, because linux wheels are build in isolated from host docker containers. + +The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`. + +Platform-specific variants also available:
+ `CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` + +#### Examples +```yaml +# build third party library +CIBW_BEFORE_ALL: make -C third_party_lib + +# install system library +CIBW_BEFORE_ALL: yum install -y libffi-dev +``` ### `CIBW_BEFORE_BUILD` {: #before-build} > Execute a shell command preparing each wheel's build diff --git a/test/14_before_all/cibuildwheel_test.py b/test/14_before_all/cibuildwheel_test.py new file mode 100644 index 00000000..e5340e35 --- /dev/null +++ b/test/14_before_all/cibuildwheel_test.py @@ -0,0 +1,23 @@ +import os + +import utils + + +def test(): + project_dir = os.path.dirname(__file__) + + with open(os.path.join(project_dir, "text_info.txt"), mode='w') as ff: + print("dummy text", file=ff) + + # build the wheels + actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + # write python version information to a temporary file, this is + # checked in setup.py + 'CIBW_BEFORE_ALL': '''python -c "open('{project}/text_info.txt', 'w').write('sample text')"''', + }) + + + # also check that we got the right wheels + os.remove(os.path.join(project_dir, "text_info.txt")) + expected_wheels = utils.expected_wheels('spam', '0.1.0') + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/14_before_all/setup.py b/test/14_before_all/setup.py new file mode 100644 index 00000000..7b68808b --- /dev/null +++ b/test/14_before_all/setup.py @@ -0,0 +1,23 @@ +import os +import sys + +from setuptools import ( + Extension, + setup, +) + +# assert that the Python version as written to text_info.txt in the CIBW_BEFORE_ALL step +# is the same one as is currently running. +with open("text_info.txt") as f: + stored_text = f.read() + +print("## stored text: "+stored_text) +assert stored_text == "sample text" + + + +setup( + name="spam", + ext_modules=[Extension('spam', sources=['spam.c'])], + version="0.1.0", +) diff --git a/test/14_before_all/spam.c b/test/14_before_all/spam.c new file mode 100644 index 00000000..d1ab0f22 --- /dev/null +++ b/test/14_before_all/spam.c @@ -0,0 +1,48 @@ +#include + +static PyObject * +spam_system(PyObject *self, PyObject *args) +{ + const char *command; + int sts; + + if (!PyArg_ParseTuple(args, "s", &command)) + return NULL; + sts = system(command); + return PyLong_FromLong(sts); +} + +/* Module initialization */ + +#if PY_MAJOR_VERSION >= 3 + #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) + #define MOD_DEF(m, name, doc, methods, module_state_size) \ + static struct PyModuleDef moduledef = { \ + PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \ + m = PyModule_Create(&moduledef); + #define MOD_RETURN(m) return m; +#else + #define MOD_INIT(name) PyMODINIT_FUNC init##name(void) + #define MOD_DEF(m, name, doc, methods, module_state_size) \ + m = Py_InitModule3(name, methods, doc); + #define MOD_RETURN(m) return; +#endif + +static PyMethodDef module_methods[] = { + {"system", (PyCFunction)spam_system, METH_VARARGS, + "Execute a shell command."}, + {NULL} /* Sentinel */ +}; + +MOD_INIT(spam) +{ + PyObject* m; + + MOD_DEF(m, + "spam", + "Example module", + module_methods, + -1) + + MOD_RETURN(m) +} From 75a0384d027f5b2c5fe16945d652a2c3dee31d58 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Mon, 11 May 2020 17:28:47 +0200 Subject: [PATCH 02/15] fix flake8 errors --- test/14_before_all/cibuildwheel_test.py | 1 - test/14_before_all/setup.py | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/test/14_before_all/cibuildwheel_test.py b/test/14_before_all/cibuildwheel_test.py index e5340e35..d5be005a 100644 --- a/test/14_before_all/cibuildwheel_test.py +++ b/test/14_before_all/cibuildwheel_test.py @@ -16,7 +16,6 @@ def test(): 'CIBW_BEFORE_ALL': '''python -c "open('{project}/text_info.txt', 'w').write('sample text')"''', }) - # also check that we got the right wheels os.remove(os.path.join(project_dir, "text_info.txt")) expected_wheels = utils.expected_wheels('spam', '0.1.0') diff --git a/test/14_before_all/setup.py b/test/14_before_all/setup.py index 7b68808b..d1cf0f48 100644 --- a/test/14_before_all/setup.py +++ b/test/14_before_all/setup.py @@ -1,6 +1,3 @@ -import os -import sys - from setuptools import ( Extension, setup, @@ -11,11 +8,10 @@ from setuptools import ( with open("text_info.txt") as f: stored_text = f.read() -print("## stored text: "+stored_text) +print("## stored text: " + stored_text) assert stored_text == "sample text" - setup( name="spam", ext_modules=[Extension('spam', sources=['spam.c'])], From 3fe560cf14a13508d9e46585d85f92bdb5bc696a Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Mon, 11 May 2020 18:04:09 +0200 Subject: [PATCH 03/15] fix mypy --- cibuildwheel/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index f7201462..012afa1a 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -121,7 +121,7 @@ class BuildOptions(NamedTuple): before_test: Optional[str] test_requires: List[str] test_extras: str -x build_verbosity: int + build_verbosity: int resources_dir = Path(__file__).resolve().parent / 'resources' From c1f155c0f2e506f1f8553a8a72ab9bbefcc82ade Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Fri, 15 May 2020 13:13:13 +0200 Subject: [PATCH 04/15] add environment evaluation --- cibuildwheel/linux.py | 16 +++++++++++++++- cibuildwheel/macos.py | 3 ++- cibuildwheel/windows.py | 3 ++- docs/options.md | 2 ++ test/14_before_all/cibuildwheel_test.py | 3 ++- test/14_before_all/setup.py | 2 +- 6 files changed, 24 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 8bbda24d..aa672f7b 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -135,7 +135,21 @@ def build(options: BuildOptions) -> None: if options.before_all: before_all_prepared = prepare_command(options.before_all, project='/project', package=options.package_dir) - call(['docker', 'exec', '-i', container_name] + shell_cmd, universal_newlines=True, input=before_all_prepared) + task = """ + PS4=' + ' + + set -o errexit + set -o xtrace + + {environment_exports} + + {before_all_prepared} + + """.format( + environment_exports='\n'.join(options.environment.as_shell_commands()), + before_all_prepared=before_all_prepared + ) + call(['docker', 'exec', '-i', container_name] + shell_cmd, universal_newlines=True, input=task) for config in platform_configs: if options.dependency_constraints: diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 3e67b534..d9d892b9 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -177,8 +177,9 @@ def build(options: BuildOptions) -> None: repaired_wheel_dir = temp_dir / 'repaired_wheel' if options.before_all: + env = options.environment.as_dictionary(prev_environment=os.environ) before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) - call([before_all_prepared], shell=True) + call([before_all_prepared], shell=True, env=env) python_configurations = get_python_configurations(options.build_selector) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 73628c7f..bae831de 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -163,8 +163,9 @@ def build(options: BuildOptions) -> None: download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) if options.before_all: + env = options.environment.as_dictionary(prev_environment=os.environ) before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) - shell([before_all_prepared]) + shell([before_all_prepared], env=env) python_configurations = get_python_configurations(options.build_selector) for config in python_configurations: diff --git a/docs/options.md b/docs/options.md index 0d61060d..ad2326d8 100644 --- a/docs/options.md +++ b/docs/options.md @@ -198,6 +198,8 @@ This option is added mainly for linux build, because linux wheels are build in i The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`. +For windows and macos `CIBW_BEFORE_ALL` python interpreter is same as `cibuildwheel` is run. For linux it is default interpreter for docker image. + Platform-specific variants also available:
`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` diff --git a/test/14_before_all/cibuildwheel_test.py b/test/14_before_all/cibuildwheel_test.py index d5be005a..1b7c8d03 100644 --- a/test/14_before_all/cibuildwheel_test.py +++ b/test/14_before_all/cibuildwheel_test.py @@ -13,7 +13,8 @@ def test(): actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ # write python version information to a temporary file, this is # checked in setup.py - 'CIBW_BEFORE_ALL': '''python -c "open('{project}/text_info.txt', 'w').write('sample text')"''', + 'CIBW_BEFORE_ALL': '''python -c "import os;open('{project}/text_info.txt', 'w').write('sample text '+os.environ.get('TEST_VAL', ''))"''', + 'CIBW_ENVIRONMENT': "TEST_VAL='123'" }) # also check that we got the right wheels diff --git a/test/14_before_all/setup.py b/test/14_before_all/setup.py index d1cf0f48..bdcadcaa 100644 --- a/test/14_before_all/setup.py +++ b/test/14_before_all/setup.py @@ -9,7 +9,7 @@ with open("text_info.txt") as f: stored_text = f.read() print("## stored text: " + stored_text) -assert stored_text == "sample text" +assert stored_text == "sample text 123" setup( From 4d5cf4629e2a320986c80a02d15474b01ec33f7f Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Fri, 15 May 2020 13:46:59 +0200 Subject: [PATCH 05/15] fix mypy type warning --- cibuildwheel/environment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index 086a253c..d0e14a02 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -1,6 +1,6 @@ import bashlex # type: ignore -from typing import Dict, List +from typing import Dict, List, MutableMapping from . import bashlex_eval @@ -61,8 +61,8 @@ class ParsedEnvironment: def __init__(self, assignments: List[EnvironmentAssignment]): self.assignments = assignments - def as_dictionary(self, prev_environment: Dict[str, str]) -> Dict[str, str]: - environment = prev_environment.copy() + def as_dictionary(self, prev_environment: MutableMapping[str, str]) -> Dict[str, str]: + environment = dict(**prev_environment) for assignment in self.assignments: value = assignment.evaluated_value(environment=environment) From 36b8f7fff252a75f0b371c4cd8caf4a71c651eaf Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 17 May 2020 17:16:23 +0200 Subject: [PATCH 06/15] changes in code from code review --- cibuildwheel/environment.py | 4 ++-- cibuildwheel/linux.py | 24 +++++++++++++----------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index d0e14a02..9f7ea2c4 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -1,6 +1,6 @@ import bashlex # type: ignore -from typing import Dict, List, MutableMapping +from typing import Dict, List, Mapping from . import bashlex_eval @@ -61,7 +61,7 @@ class ParsedEnvironment: def __init__(self, assignments: List[EnvironmentAssignment]): self.assignments = assignments - def as_dictionary(self, prev_environment: MutableMapping[str, str]) -> Dict[str, str]: + def as_dictionary(self, prev_environment: Mapping[str, str]) -> Dict[str, str]: environment = dict(**prev_environment) for assignment in self.assignments: diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index aa672f7b..aa620f21 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -134,22 +134,24 @@ def build(options: BuildOptions) -> None: call(['docker', 'start', container_name]) if options.before_all: - before_all_prepared = prepare_command(options.before_all, project='/project', package=options.package_dir) - task = """ - PS4=' + ' + call( + ['docker', 'exec', '-i', container_name] + shell_cmd, + universal_newlines=True, + input=''' + PS4=' + ' - set -o errexit - set -o xtrace + set -o errexit + set -o xtrace - {environment_exports} + {environment_exports} - {before_all_prepared} + sh -c {before_all} - """.format( - environment_exports='\n'.join(options.environment.as_shell_commands()), - before_all_prepared=before_all_prepared + '''.format( + environment_exports='\n'.join(options.environment.as_shell_commands()), + before_all=prepare_command(options.before_all, project='/project', package=container_package_dir) + ) ) - call(['docker', 'exec', '-i', container_name] + shell_cmd, universal_newlines=True, input=task) for config in platform_configs: if options.dependency_constraints: From 219a65d5e380dd96361741f3d102eaf64d4749b2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 17 May 2020 17:14:50 +0200 Subject: [PATCH 07/15] Apply suggestions from code review Co-authored-by: Joe Rickerby --- docs/options.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/options.md b/docs/options.md index ad2326d8..5fbde070 100644 --- a/docs/options.md +++ b/docs/options.md @@ -191,14 +191,15 @@ CIBW_ENVIRONMENT: "BUILD_TIME=$(date) SAMPLE_TEXT=\"sample text\"" `cibuildwheel` always defines the environment variable `CIBUILDWHEEL=1`. This can be useful for [building wheels with optional extensions](faq.md#building-packages-with-optional-c-extensions). ### `CIBW_BEFORE_ALL` {: #before-all} -> Execute a shell command preparing common part for each wheel. +> Execute a shell command on the build system before any wheels are built. -Shell command to prepare common part of project (ex. build libraries which does not depend on python). -This option is added mainly for linux build, because linux wheels are build in isolated from host docker containers. +Shell command to prepare a common part of the project (e.g. build libraries which does not depend on the specific version of Python). + +This option is very useful for the Linux build, where builds take place in isolated Docker containers managed by cibuildwheel. This command will run inside the container before the wheel builds start. Note, if you're building both x86_64 and i686 wheels (the default), your build uses two different Docker images. In that case, this command will execute twice - once per build container. The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`. -For windows and macos `CIBW_BEFORE_ALL` python interpreter is same as `cibuildwheel` is run. For linux it is default interpreter for docker image. +The version of Python available inside `CIBW_BEFORE_ALL` is not controlled by cibuildwheel. On Windows and macOS, it's whatever is available on the host machine. For Linux, this comes from the Docker image, which on manylinux, is something rather old. Consider using a Python inside `/opt/python`, e.g. `/opt/python/cp38-cp38/bin/python`. Platform-specific variants also available:
`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` @@ -209,7 +210,7 @@ Platform-specific variants also available:
CIBW_BEFORE_ALL: make -C third_party_lib # install system library -CIBW_BEFORE_ALL: yum install -y libffi-dev +CIBW_BEFORE_ALL_LINUX: yum install -y libffi-dev ``` ### `CIBW_BEFORE_BUILD` {: #before-build} From 9f6c91404dc5910ee142fa35097c78f48261f9be Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 17 May 2020 17:41:53 +0200 Subject: [PATCH 08/15] fix flake8 --- cibuildwheel/linux.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index aa620f21..02cd28c8 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -135,7 +135,7 @@ def build(options: BuildOptions) -> None: if options.before_all: call( - ['docker', 'exec', '-i', container_name] + shell_cmd, + ['docker', 'exec', '-i', container_name] + shell_cmd, universal_newlines=True, input=''' PS4=' + ' @@ -147,10 +147,10 @@ def build(options: BuildOptions) -> None: sh -c {before_all} - '''.format( - environment_exports='\n'.join(options.environment.as_shell_commands()), - before_all=prepare_command(options.before_all, project='/project', package=container_package_dir) - ) + '''.format( + environment_exports='\n'.join(options.environment.as_shell_commands()), + before_all=prepare_command(options.before_all, project='/project', package=container_package_dir) + ) ) for config in platform_configs: From 7b7e24d36da160c8d17ab6ceada66f4bee248e05 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 17 May 2020 18:45:15 +0200 Subject: [PATCH 09/15] add missed quote of before_all command --- cibuildwheel/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 02cd28c8..3def204b 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -149,7 +149,7 @@ def build(options: BuildOptions) -> None: '''.format( environment_exports='\n'.join(options.environment.as_shell_commands()), - before_all=prepare_command(options.before_all, project='/project', package=container_package_dir) + before_all=shlex.quote(prepare_command(options.before_all, project='/project', package=container_package_dir)) ) ) From fc52c5e7b6d19e4e220a36b1cd960380981f68a3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 17 May 2020 18:53:42 +0200 Subject: [PATCH 10/15] add unit_test --- unit_test/main_tests/main_options_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index 14252e67..b0422589 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -233,3 +233,18 @@ def test_build_selector_migrations(intercepted_build_args, monkeypatch, option_n assert intercepted_build_selector.build_patterns == build_selector_patterns else: assert intercepted_build_selector.skip_patterns == build_selector_patterns + + +@pytest.mark.parametrize('before_all', [None, 'test text']) +@pytest.mark.parametrize('platform_specific', [False, True]) +def test_before_all(before_all, platform_specific, platform, intercepted_build_args, monkeypatch): + if before_all is not None: + if platform_specific: + monkeypatch.setenv('CIBW_BEFORE_ALL_' + platform.upper(), before_all) + monkeypatch.setenv('CIBW_BEFORE_ALL', 'overwritten') + else: + monkeypatch.setenv('CIBW_BEFORE_ALL', before_all) + + main() + + assert intercepted_build_args.args[0].before_all == before_all From 2e15fdddfd9e44c07d573d18a7e12e914036dbde Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Mon, 18 May 2020 10:42:13 +0200 Subject: [PATCH 11/15] change to new test engine --- test/14_before_all/setup.py | 19 -------- test/14_before_all/spam.c | 48 ------------------- ...ibuildwheel_test.py => test_before_all.py} | 21 ++++++-- 3 files changed, 18 insertions(+), 70 deletions(-) delete mode 100644 test/14_before_all/setup.py delete mode 100644 test/14_before_all/spam.c rename test/{14_before_all/cibuildwheel_test.py => test_before_all.py} (54%) diff --git a/test/14_before_all/setup.py b/test/14_before_all/setup.py deleted file mode 100644 index bdcadcaa..00000000 --- a/test/14_before_all/setup.py +++ /dev/null @@ -1,19 +0,0 @@ -from setuptools import ( - Extension, - setup, -) - -# assert that the Python version as written to text_info.txt in the CIBW_BEFORE_ALL step -# is the same one as is currently running. -with open("text_info.txt") as f: - stored_text = f.read() - -print("## stored text: " + stored_text) -assert stored_text == "sample text 123" - - -setup( - name="spam", - ext_modules=[Extension('spam', sources=['spam.c'])], - version="0.1.0", -) diff --git a/test/14_before_all/spam.c b/test/14_before_all/spam.c deleted file mode 100644 index d1ab0f22..00000000 --- a/test/14_before_all/spam.c +++ /dev/null @@ -1,48 +0,0 @@ -#include - -static PyObject * -spam_system(PyObject *self, PyObject *args) -{ - const char *command; - int sts; - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - sts = system(command); - return PyLong_FromLong(sts); -} - -/* Module initialization */ - -#if PY_MAJOR_VERSION >= 3 - #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) - #define MOD_DEF(m, name, doc, methods, module_state_size) \ - static struct PyModuleDef moduledef = { \ - PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \ - m = PyModule_Create(&moduledef); - #define MOD_RETURN(m) return m; -#else - #define MOD_INIT(name) PyMODINIT_FUNC init##name(void) - #define MOD_DEF(m, name, doc, methods, module_state_size) \ - m = Py_InitModule3(name, methods, doc); - #define MOD_RETURN(m) return; -#endif - -static PyMethodDef module_methods[] = { - {"system", (PyCFunction)spam_system, METH_VARARGS, - "Execute a shell command."}, - {NULL} /* Sentinel */ -}; - -MOD_INIT(spam) -{ - PyObject* m; - - MOD_DEF(m, - "spam", - "Example module", - module_methods, - -1) - - MOD_RETURN(m) -} diff --git a/test/14_before_all/cibuildwheel_test.py b/test/test_before_all.py similarity index 54% rename from test/14_before_all/cibuildwheel_test.py rename to test/test_before_all.py index 1b7c8d03..177ec555 100644 --- a/test/14_before_all/cibuildwheel_test.py +++ b/test/test_before_all.py @@ -1,10 +1,25 @@ import os +import textwrap -import utils +from . import utils +from . import test_projects + +project_with_before_build_asserts = test_projects.new_c_project( + setup_py_add=textwrap.dedent(r''' + # assert that the Python version as written to text_info.txt in the CIBW_BEFORE_ALL step + # is the same one as is currently running. + with open("text_info.txt") as f: + stored_text = f.read() + + print("## stored text: " + stored_text) + assert stored_text == "sample text 123" + ''') +) -def test(): - project_dir = os.path.dirname(__file__) +def test(tmp_path): + project_dir = tmp_path / 'project' + project_with_before_build_asserts.generate(project_dir) with open(os.path.join(project_dir, "text_info.txt"), mode='w') as ff: print("dummy text", file=ff) From 46352c8e309973cb44dd84168b1c444f33d7815d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 14 Jun 2020 10:34:44 +0100 Subject: [PATCH 12/15] Add a modern python to linux build before_all environment --- cibuildwheel/linux.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 3def204b..6e42f47f 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -143,6 +143,10 @@ def build(options: BuildOptions) -> None: set -o errexit set -o xtrace + # add a modern Python interpreter to PATH so it can be used by BEFORE_ALL + # commands + export PATH=/opt/python/cp38-cp38:$PATH + {environment_exports} sh -c {before_all} From 2057bba07f0b78b838d506e9a167e13875c94d8e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 14 Jun 2020 10:35:54 +0100 Subject: [PATCH 13/15] Update docs to reflect linux available python change --- docs/options.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/options.md b/docs/options.md index 5fbde070..aba121db 100644 --- a/docs/options.md +++ b/docs/options.md @@ -193,13 +193,13 @@ CIBW_ENVIRONMENT: "BUILD_TIME=$(date) SAMPLE_TEXT=\"sample text\"" ### `CIBW_BEFORE_ALL` {: #before-all} > Execute a shell command on the build system before any wheels are built. -Shell command to prepare a common part of the project (e.g. build libraries which does not depend on the specific version of Python). +Shell command to prepare a common part of the project (e.g. build or install libraries which does not depend on the specific version of Python). This option is very useful for the Linux build, where builds take place in isolated Docker containers managed by cibuildwheel. This command will run inside the container before the wheel builds start. Note, if you're building both x86_64 and i686 wheels (the default), your build uses two different Docker images. In that case, this command will execute twice - once per build container. The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`. -The version of Python available inside `CIBW_BEFORE_ALL` is not controlled by cibuildwheel. On Windows and macOS, it's whatever is available on the host machine. For Linux, this comes from the Docker image, which on manylinux, is something rather old. Consider using a Python inside `/opt/python`, e.g. `/opt/python/cp38-cp38/bin/python`. +On Windows and macOS, the version of Python available inside `CIBW_BEFORE_ALL` is whatever is available on the host machine. On Linux, a modern Python version is available on PATH. Platform-specific variants also available:
`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` From b4b46a1f066e4657e75ab4ac392f0d4076a5a704 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 14 Jun 2020 21:35:28 +0200 Subject: [PATCH 14/15] fix mypy --- cibuildwheel/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index bce3f754..4e25befa 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -128,7 +128,7 @@ def main() -> None: build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '') environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='') - before_all = get_option_from_environment('CIBW_BEFORE_ALL', platform=platform) + before_all = get_option_from_environment('CIBW_BEFORE_ALL', platform=platform, default='') before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default) dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned') From 43fcb31231ee540686c5dd0e3be67d6fbb816911 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Mon, 22 Jun 2020 13:52:56 +0200 Subject: [PATCH 15/15] fix tests --- unit_test/main_tests/main_options_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index b0422589..c7e6bfdd 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -235,7 +235,7 @@ def test_build_selector_migrations(intercepted_build_args, monkeypatch, option_n assert intercepted_build_selector.skip_patterns == build_selector_patterns -@pytest.mark.parametrize('before_all', [None, 'test text']) +@pytest.mark.parametrize('before_all', ["", None, 'test text']) @pytest.mark.parametrize('platform_specific', [False, True]) def test_before_all(before_all, platform_specific, platform, intercepted_build_args, monkeypatch): if before_all is not None: @@ -247,4 +247,7 @@ def test_before_all(before_all, platform_specific, platform, intercepted_build_a main() + if before_all is None: + before_all = "" + assert intercepted_build_args.args[0].before_all == before_all