From edd461ad42a6ffb835c4335796f8c6b7f84cdbc2 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 17 May 2020 22:26:12 +0200 Subject: [PATCH 1/3] Making order of options more consistent in docs and BuildOptions fields --- README.md | 4 +-- cibuildwheel/__main__.py | 4 +-- cibuildwheel/util.py | 16 +++++------ docs/options.md | 60 +++++++++++++++++++++------------------- 4 files changed, 43 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 93c896ce..78e6a2f8 100644 --- a/README.md +++ b/README.md @@ -105,10 +105,10 @@ Options | | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build | | | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each (non-pure Python) built wheel | | | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_PYPY_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) | Specify alternative manylinux docker images | -| **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command) | Execute a shell command to test each built wheel | -| | [`CIBW_BEFORE_TEST`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-test) | Execute shell command to prepare test environment | +| **Testing** | [`CIBW_BEFORE_TEST`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-test) | Execute shell command to prepare test environment | | | [`CIBW_TEST_REQUIRES`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-requires) | Install Python dependencies before running the tests | | | [`CIBW_TEST_EXTRAS`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras) | Install your wheel for testing using extras_require | +| | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command) | Execute a shell command to test each built wheel | | **Other** | [`CIBW_BUILD_VERBOSITY`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-verbosity) | Increase/decrease the output of pip wheel | Working examples diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index bc21fbe6..cb3b5349 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -115,9 +115,10 @@ def main() -> None: exit(2) output_dir = args.output_dir - test_command = get_option_from_environment('CIBW_TEST_COMMAND', platform=platform) test_requires = get_option_from_environment('CIBW_TEST_REQUIRES', platform=platform, default='').split() test_extras = get_option_from_environment('CIBW_TEST_EXTRAS', platform=platform, default='') + before_test = get_option_from_environment('CIBW_BEFORE_TEST', platform=platform, default='') + test_command = get_option_from_environment('CIBW_TEST_COMMAND', platform=platform) package_dir = args.package_dir before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) build_verbosity_str = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='') @@ -130,7 +131,6 @@ def main() -> None: repair_command_default = '' repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default) environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='') - before_test = get_option_from_environment('CIBW_BEFORE_TEST', platform=platform, default='') dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned') if dependency_versions == 'pinned': diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 80042039..0742fc3f 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -110,17 +110,17 @@ class DependencyConstraints: class BuildOptions(NamedTuple): package_dir: str output_dir: str - test_command: Optional[str] + build_selector: BuildSelector + environment: ParsedEnvironment + before_build: Optional[str] + repair_command: str + manylinux_images: Optional[Dict[str, str]] + dependency_constraints: Optional[DependencyConstraints] + before_test: str test_requires: List[str] test_extras: str - before_build: Optional[str] + test_command: Optional[str] build_verbosity: int - build_selector: BuildSelector - repair_command: str - environment: ParsedEnvironment - before_test: str - dependency_constraints: Optional[DependencyConstraints] - manylinux_images: Optional[Dict[str, str]] resources_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources')) diff --git a/docs/options.md b/docs/options.md index 8ecaaf23..2e8dd324 100644 --- a/docs/options.md +++ b/docs/options.md @@ -52,7 +52,6 @@ env: ## Build selection - ### `CIBW_PLATFORM` {: #platform} > Override the auto-detected target platform @@ -67,6 +66,7 @@ For `linux` you need Docker running, on macOS or Linux. For `macos`, you need a This option can also be set using the command-line option `--platform`. + ### `CIBW_BUILD`, `CIBW_SKIP` {: #build-skip} > Choose the Python versions to build @@ -155,7 +155,6 @@ CIBW_SKIP: pp* ## Build customization - ### `CIBW_ENVIRONMENT` {: #environment} > Set environment variables needed during the build @@ -296,6 +295,7 @@ CIBW_MANYLINUX_X86_64_IMAGE: dockcross/manylinux-x64 CIBW_MANYLINUX_I686_IMAGE: dockcross/manylinux-x86 ``` + ### `CIBW_DEPENDENCY_VERSIONS` {: #dependency-versions} > Specify how cibuildwheel controls the versions of the tools it uses @@ -348,27 +348,32 @@ CIBW_DEPENDENCY_VERSIONS: ./constraints.txt ## Testing -### `CIBW_TEST_COMMAND` {: #test-command} -> Execute a shell command to test each built wheel +### `CIBW_BEFORE_TEST` {: #before-test} +> Execute a shell command before testing each wheel -Shell command to run tests after the build. The wheel will be installed automatically and available for import from the tests. To ensure the wheel is imported by your tests (instead of your source copy), tests are run from a different directory. Use the placeholders `{project}` and `{package}` when specifying paths in your project. +A shell command to run in **each** test virtual environment, before your wheel is installed and tested. This is useful if you need to install a non pip package, change values of environment variables +or perform multi step pip installation (e.g. installing `scikit-build` or `cython` before install test package) -- `{project}` is an absolute path to the project root - the working directory where cibuildwheel was called. -- `{package}` is the path to the package being built - the `package_dir` argument supplied to cibuildwheel on the command line. +The active Python binary can be accessed using `python`, and pip with `pip`; `cibuildwheel` makes sure the right version of Python and pip will be executed. The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`. The command is run in a shell, so you can write things like `cmd1 && cmd2`. Platform-specific variants also available:
-`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` + `CIBW_BEFORE_TEST_MACOS` | `CIBW_BEFORE_TEST_WINDOWS` | `CIBW_BEFORE_TEST_LINUX` #### Examples - ```yaml -# run the project tests against the installed wheel using `nose` -CIBW_TEST_COMMAND: nosetests {project}/tests +# install test dependencies with overwritten environment variables. +CIBW_BEFORE_TEST: CC=gcc CXX=g++ pip install -r requirements.txt -# run the package tests using `pytest` -CIBW_TEST_COMMAND: pytest {package}/tests +# chain commands using && +CIBW_BEFORE_TEST: rm -rf ./data/cache && mkdir -p ./data/cache + +# install non pip python package +CIBW_BEFORE_TEST: cd some_dir; ./configure; make; make install + +# install python packages that are required to install test dependencies +CIBW_BEFORE_TEST: pip install cmake scikit-build ``` @@ -411,34 +416,31 @@ Platform-specific variants also available:
CIBW_TEST_EXTRAS: test,qt ``` -### `CIBW_BEFORE_TEST` {: #before-test} -> Execute a shell command before testing each wheel -A shell command to run in **each** test virtual environment, before your wheel is installed and tested. This is useful if you need to install a non pip package, change values of environment variables -or perform multi step pip installation (e.g. installing `scikit-build` or `cython` before install test package) +### `CIBW_TEST_COMMAND` {: #test-command} +> Execute a shell command to test each built wheel -The active Python binary can be accessed using `python`, and pip with `pip`; `cibuildwheel` makes sure the right version of Python and pip will be executed. The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`. +Shell command to run tests after the build. The wheel will be installed automatically and available for import from the tests. To ensure the wheel is imported by your tests (instead of your source copy), tests are run from a different directory. Use the placeholders `{project}` and `{package}` when specifying paths in your project. + +- `{project}` is an absolute path to the project root - the working directory where cibuildwheel was called. +- `{package}` is the path to the package being built - the `package_dir` argument supplied to cibuildwheel on the command line. The command is run in a shell, so you can write things like `cmd1 && cmd2`. Platform-specific variants also available:
- `CIBW_BEFORE_TEST_MACOS` | `CIBW_BEFORE_TEST_WINDOWS` | `CIBW_BEFORE_TEST_LINUX` +`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` #### Examples + ```yaml -# install test dependencies with overwritten environment variables. -CIBW_BEFORE_TEST: CC=gcc CXX=g++ pip install -r requirements.txt +# run the project tests against the installed wheel using `nose` +CIBW_TEST_COMMAND: nosetests {project}/tests -# chain commands using && -CIBW_BEFORE_TEST: rm -rf ./data/cache && mkdir -p ./data/cache - -# install non pip python package -CIBW_BEFORE_TEST: cd some_dir; ./configure; make; make install - -# install python packages that are required to install test dependencies -CIBW_BEFORE_TEST: pip install cmake scikit-build +# run the package tests using `pytest` +CIBW_TEST_COMMAND: pytest {package}/tests ``` + ## Other ### `CIBW_BUILD_VERBOSITY` {: #build-verbosity} From 2790fd1dd7d37491c42001ae8f644a526612d466 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 31 May 2020 21:42:21 +0200 Subject: [PATCH 2/3] Put CIBW_TEST_COMMAND as first test option everywhere --- README.md | 4 ++-- cibuildwheel/__main__.py | 37 ++++++++++++++++--------------- cibuildwheel/util.py | 4 ++-- docs/options.md | 48 ++++++++++++++++++++-------------------- 4 files changed, 47 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 78e6a2f8..93c896ce 100644 --- a/README.md +++ b/README.md @@ -105,10 +105,10 @@ Options | | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build | | | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each (non-pure Python) built wheel | | | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_PYPY_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) | Specify alternative manylinux docker images | -| **Testing** | [`CIBW_BEFORE_TEST`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-test) | Execute shell command to prepare test environment | +| **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command) | Execute a shell command to test each built wheel | +| | [`CIBW_BEFORE_TEST`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-test) | Execute shell command to prepare test environment | | | [`CIBW_TEST_REQUIRES`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-requires) | Install Python dependencies before running the tests | | | [`CIBW_TEST_EXTRAS`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras) | Install your wheel for testing using extras_require | -| | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command) | Execute a shell command to test each built wheel | | **Other** | [`CIBW_BUILD_VERBOSITY`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-verbosity) | Increase/decrease the output of pip wheel | Working examples diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index cb3b5349..53ac3870 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -114,15 +114,21 @@ def main() -> None: file=sys.stderr) exit(2) - output_dir = args.output_dir - test_requires = get_option_from_environment('CIBW_TEST_REQUIRES', platform=platform, default='').split() - test_extras = get_option_from_environment('CIBW_TEST_EXTRAS', platform=platform, default='') - before_test = get_option_from_environment('CIBW_BEFORE_TEST', platform=platform, default='') - test_command = get_option_from_environment('CIBW_TEST_COMMAND', platform=platform) package_dir = args.package_dir - before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) - build_verbosity_str = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='') + output_dir = args.output_dir + build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '') + build_selector = BuildSelector(build_config, skip_config) + + environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='') + try: + environment = parse_environment(environment_config) + except (EnvironmentParseError, ValueError): + print(f'cibuildwheel: Malformed environment option "{environment_config}"', file=sys.stderr) + traceback.print_exc(None, sys.stderr) + exit(2) + + before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) if platform == 'linux': repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}' elif platform == 'macos': @@ -130,7 +136,6 @@ def main() -> None: else: repair_command_default = '' repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default) - environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='') dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned') if dependency_versions == 'pinned': @@ -140,23 +145,19 @@ def main() -> None: else: dependency_constraints = DependencyConstraints(dependency_versions) + test_command = get_option_from_environment('CIBW_TEST_COMMAND', platform=platform) + before_test = get_option_from_environment('CIBW_BEFORE_TEST', platform=platform) + test_requires = get_option_from_environment('CIBW_TEST_REQUIRES', platform=platform, default='').split() + test_extras = get_option_from_environment('CIBW_TEST_EXTRAS', platform=platform, default='') if test_extras: test_extras = f'[{test_extras}]' + build_verbosity_str = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='') try: build_verbosity = min(3, max(-3, int(build_verbosity_str))) except ValueError: build_verbosity = 0 - try: - environment = parse_environment(environment_config) - except (EnvironmentParseError, ValueError): - print(f'cibuildwheel: Malformed environment option "{environment_config}"', file=sys.stderr) - traceback.print_exc(None, sys.stderr) - exit(2) - - build_selector = BuildSelector(build_config, skip_config) - # Add CIBUILDWHEEL environment variable # This needs to be passed on to the docker container in linux.py os.environ['CIBUILDWHEEL'] = '1' @@ -206,12 +207,12 @@ def main() -> None: test_command=test_command, test_requires=test_requires, test_extras=test_extras, + before_test=before_test, before_build=before_build, build_verbosity=build_verbosity, build_selector=build_selector, repair_command=repair_command, environment=environment, - before_test=before_test, dependency_constraints=dependency_constraints, manylinux_images=manylinux_images, ) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 0742fc3f..499954b9 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -116,10 +116,10 @@ class BuildOptions(NamedTuple): repair_command: str manylinux_images: Optional[Dict[str, str]] dependency_constraints: Optional[DependencyConstraints] - before_test: str + test_command: Optional[str] + before_test: Optional[str] test_requires: List[str] test_extras: str - test_command: Optional[str] build_verbosity: int diff --git a/docs/options.md b/docs/options.md index 2e8dd324..18d5417b 100644 --- a/docs/options.md +++ b/docs/options.md @@ -348,6 +348,30 @@ CIBW_DEPENDENCY_VERSIONS: ./constraints.txt ## Testing +### `CIBW_TEST_COMMAND` {: #test-command} +> Execute a shell command to test each built wheel + +Shell command to run tests after the build. The wheel will be installed automatically and available for import from the tests. To ensure the wheel is imported by your tests (instead of your source copy), tests are run from a different directory. Use the placeholders `{project}` and `{package}` when specifying paths in your project. + +- `{project}` is an absolute path to the project root - the working directory where cibuildwheel was called. +- `{package}` is the path to the package being built - the `package_dir` argument supplied to cibuildwheel on the command line. + +The command is run in a shell, so you can write things like `cmd1 && cmd2`. + +Platform-specific variants also available:
+`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` + +#### Examples + +```yaml +# run the project tests against the installed wheel using `nose` +CIBW_TEST_COMMAND: nosetests {project}/tests + +# run the package tests using `pytest` +CIBW_TEST_COMMAND: pytest {package}/tests +``` + + ### `CIBW_BEFORE_TEST` {: #before-test} > Execute a shell command before testing each wheel @@ -417,30 +441,6 @@ CIBW_TEST_EXTRAS: test,qt ``` -### `CIBW_TEST_COMMAND` {: #test-command} -> Execute a shell command to test each built wheel - -Shell command to run tests after the build. The wheel will be installed automatically and available for import from the tests. To ensure the wheel is imported by your tests (instead of your source copy), tests are run from a different directory. Use the placeholders `{project}` and `{package}` when specifying paths in your project. - -- `{project}` is an absolute path to the project root - the working directory where cibuildwheel was called. -- `{package}` is the path to the package being built - the `package_dir` argument supplied to cibuildwheel on the command line. - -The command is run in a shell, so you can write things like `cmd1 && cmd2`. - -Platform-specific variants also available:
-`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` - -#### Examples - -```yaml -# run the project tests against the installed wheel using `nose` -CIBW_TEST_COMMAND: nosetests {project}/tests - -# run the package tests using `pytest` -CIBW_TEST_COMMAND: pytest {package}/tests -``` - - ## Other ### `CIBW_BUILD_VERBOSITY` {: #build-verbosity} From e97b103148dac86d113ccd0d682cacedb5beb460 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 1 Jun 2020 00:01:57 +0200 Subject: [PATCH 3/3] Group environment variables querying in __main__.py --- cibuildwheel/__main__.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 53ac3870..00a62a7f 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -117,10 +117,26 @@ def main() -> None: package_dir = args.package_dir output_dir = args.output_dir + if platform == 'linux': + repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}' + elif platform == 'macos': + repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}' + else: + repair_command_default = '' + build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '') + environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='') + before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) + repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default) + dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned') + test_command = get_option_from_environment('CIBW_TEST_COMMAND', platform=platform) + before_test = get_option_from_environment('CIBW_BEFORE_TEST', platform=platform) + test_requires = get_option_from_environment('CIBW_TEST_REQUIRES', platform=platform, default='').split() + test_extras = get_option_from_environment('CIBW_TEST_EXTRAS', platform=platform, default='') + build_verbosity_str = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='') + build_selector = BuildSelector(build_config, skip_config) - environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='') try: environment = parse_environment(environment_config) except (EnvironmentParseError, ValueError): @@ -128,16 +144,6 @@ def main() -> None: traceback.print_exc(None, sys.stderr) exit(2) - before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) - if platform == 'linux': - repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}' - elif platform == 'macos': - repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}' - else: - repair_command_default = '' - repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default) - - dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned') if dependency_versions == 'pinned': dependency_constraints: Optional[DependencyConstraints] = DependencyConstraints.with_defaults() elif dependency_versions == 'latest': @@ -145,14 +151,9 @@ def main() -> None: else: dependency_constraints = DependencyConstraints(dependency_versions) - test_command = get_option_from_environment('CIBW_TEST_COMMAND', platform=platform) - before_test = get_option_from_environment('CIBW_BEFORE_TEST', platform=platform) - test_requires = get_option_from_environment('CIBW_TEST_REQUIRES', platform=platform, default='').split() - test_extras = get_option_from_environment('CIBW_TEST_EXTRAS', platform=platform, default='') if test_extras: test_extras = f'[{test_extras}]' - build_verbosity_str = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='') try: build_verbosity = min(3, max(-3, int(build_verbosity_str))) except ValueError: