Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #482 from asfaltboy/support-quemu-on-github

Browse Source 
Support qemu in GitHub Actions
This commit is contained in:
Joe Rickerby
2021-01-01 17:21:24 +00:00
committed by GitHub
parent eed12cf4c9 aa871284fc
commit 5c695593f3
17 changed files with 393 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/__main__.py
+24 -3
View File
@@ -17,6 +17,7 @@ from cibuildwheel.environment import (
parse_environment,
)
from cibuildwheel.util import (
Architecture,
BuildOptions,
BuildSelector,
DependencyConstraints,
@@ -67,6 +68,17 @@ def main() -> None:
run in Windows, and it will build and test for all versions of
Python. Default: auto.
''')
parser.add_argument('--archs',
default=None,
help='''
Comma-separated list of CPU architectures to build for.
When set to 'auto', builds the architectures natively supported
on this machine. Set this option to build an architecture
via emulation, for example, using binfmt_misc and QEMU.
Default: auto.
Choices: auto, {}
'''.format(", ".join(a.name for a in Architecture)))
parser.add_argument('--output-dir',
default=os.environ.get('CIBW_OUTPUT_DIR', 'wheelhouse'),
help='Destination folder for the wheels.')
@@ -168,8 +180,14 @@ def main() -> None:
print('cibuildwheel: Could not find setup.py, setup.cfg or pyproject.toml at root of package', file=sys.stderr)
exit(2)
if args.archs is not None:
archs_config_str = args.archs
else:
archs_config_str = get_option_from_environment('CIBW_ARCHS', platform=platform, default='auto')
archs = Architecture.parse_config(archs_config_str, platform=platform)
if args.print_build_identifiers:
print_build_identifiers(platform, build_selector)
print_build_identifiers(platform, build_selector, archs)
exit(0)
manylinux_images: Optional[Dict[str, str]] = None
@@ -202,6 +220,7 @@ def main() -> None:
manylinux_images[build_platform] = image
build_options = BuildOptions(
architectures=archs,
package_dir=package_dir,
output_dir=output_dir,
test_command=test_command,
@@ -284,10 +303,12 @@ def print_preamble(platform: str, build_options: BuildOptions) -> None:
print('\nHere we go!\n')
def print_build_identifiers(platform: str, build_selector: BuildSelector) -> None:
def print_build_identifiers(
platform: str, build_selector: BuildSelector, architectures: List[Architecture]
) -> None:
python_configurations: List[Any] = []
if platform == 'linux':
python_configurations = cibuildwheel.linux.get_python_configurations(build_selector)
python_configurations = cibuildwheel.linux.get_python_configurations(build_selector, architectures)
elif platform == 'windows':
python_configurations = cibuildwheel.windows.get_python_configurations(build_selector)
elif platform == 'macos':
cibuildwheel/linux.py
+16 -28
View File
@@ -1,4 +1,3 @@
import platform
import subprocess
import sys
import textwrap
@@ -8,29 +7,10 @@ from typing import List, NamedTuple, Union
from .docker_container import DockerContainer
from .logger import log
from .util import (BuildOptions, BuildSelector, NonPlatformWheelError,
get_build_verbosity_extra_flags, prepare_command)
def matches_platform(identifier: str) -> bool:
pm = platform.machine()
if pm == "x86_64":
# x86_64 machines can run i686 docker containers
if identifier.endswith('x86_64') or identifier.endswith('i686'):
return True
elif pm == "i686":
if identifier.endswith('i686'):
return True
elif pm == "aarch64":
if identifier.endswith('aarch64'):
return True
elif pm == "ppc64le":
if identifier.endswith('ppc64le'):
return True
elif pm == "s390x":
if identifier.endswith('s390x'):
return True
return False
from .util import (
Architecture, BuildOptions, BuildSelector, NonPlatformWheelError,
get_build_verbosity_extra_flags, prepare_command,
)
class PythonConfiguration(NamedTuple):
@@ -43,7 +23,9 @@ class PythonConfiguration(NamedTuple):
return PurePath(self.path_str)
def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]:
def get_python_configurations(
build_selector: BuildSelector, architectures: List[Architecture]
) -> List[PythonConfiguration]:
python_configurations = [
PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27m'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27mu'),
@@ -78,8 +60,14 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi
PythonConfiguration(version='3.8', identifier='cp38-manylinux_s390x', path_str='/opt/python/cp38-cp38'),
PythonConfiguration(version='3.9', identifier='cp39-manylinux_s390x', path_str='/opt/python/cp39-cp39'),
]
# skip builds as required
return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)]
# return all configurations whose arch is in our `architectures` list,
# and match the build/skip rules
return [
c for c in python_configurations
if any(c.identifier.endswith(arch.value) for arch in architectures)
and build_selector(c.identifier)
]
def build(options: BuildOptions) -> None:
@@ -93,7 +81,7 @@ def build(options: BuildOptions) -> None:
exit(2)
assert options.manylinux_images is not None
python_configurations = get_python_configurations(options.build_selector)
python_configurations = get_python_configurations(options.build_selector, options.architectures)
platforms = [
('cp', 'manylinux_x86_64', options.manylinux_images['x86_64']),
('cp', 'manylinux_i686', options.manylinux_images['i686']),
cibuildwheel/macos.py
+8 -1
View File
@@ -4,13 +4,14 @@ import shutil
import subprocess
import sys
import tempfile
import textwrap
from os import PathLike
from pathlib import Path
from typing import Dict, List, NamedTuple, Optional, Sequence, Union
from .environment import ParsedEnvironment
from .logger import log
from .util import (BuildOptions, BuildSelector, NonPlatformWheelError,
from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError,
download, get_build_verbosity_extra_flags, get_pip_script,
install_certifi_script, prepare_command)
@@ -186,6 +187,12 @@ def setup_python(python_configuration: PythonConfiguration,
def build(options: BuildOptions) -> None:
if options.architectures != [Architecture.x86_64]:
raise ValueError(textwrap.dedent(f'''
Invalid archs option {options.architectures}. macOS only supports x86_64 for the moment.
If you want to set emulation architectures on Linux, use CIBW_ARCHS_LINUX instead.
'''))
temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel'))
built_wheel_dir = temp_dir / 'built_wheel'
repaired_wheel_dir = temp_dir / 'repaired_wheel'
cibuildwheel/util.py
+38 -1
View File
@@ -1,4 +1,6 @@
import os
import platform as platform_module
import re
import ssl
import textwrap
import urllib.request
@@ -121,10 +123,45 @@ class DependencyConstraints:
return f'{self.__class__.__name__}{self.base_file_path!r})'
class Architecture(Enum):
# mac/linux archs
x86_64 = 'x86_64'
i686 = 'i686'
aarch64 = 'aarch64'
ppc64le = 'ppc64le'
s390x = 's390x'
# windows archs
x86 = 'x86'
AMD64 = 'AMD64'
@staticmethod
def parse_config(config: str, platform: str) -> 'List[Architecture]':
result = []
for arch_str in re.split(r'[\s,]+', config):
if arch_str == 'auto':
result += Architecture.auto_archs(platform=platform)
else:
result.append(Architecture(arch_str))
return result
@staticmethod
def auto_archs(platform: str) -> 'List[Architecture]':
native_architecture = Architecture(platform_module.machine())
result = [native_architecture]
if platform == 'linux' and native_architecture == Architecture.x86_64:
# x86_64 machines can run i686 docker containers
result.append(Architecture.i686)
if platform == 'windows' and native_architecture == Architecture.AMD64:
result.append(Architecture.x86)
return result
class BuildOptions(NamedTuple):
package_dir: Path
output_dir: Path
build_selector: BuildSelector
architectures: List[Architecture]
environment: ParsedEnvironment
before_all: str
before_build: Optional[str]
@@ -164,7 +201,7 @@ def strtobool(val: str) -> bool:
return False
class CIProvider(str, Enum):
class CIProvider(Enum):
travis_ci = 'travis'
appveyor = 'appveyor'
circle_ci = 'circle_ci'
cibuildwheel/windows.py
+9 -1
View File
@@ -3,6 +3,7 @@ import shutil
import subprocess
import sys
import tempfile
import textwrap
from os import PathLike
from pathlib import Path
from typing import Dict, List, NamedTuple, Optional, Sequence, Union
@@ -12,7 +13,7 @@ import toml
from .environment import ParsedEnvironment
from .logger import log
from .util import (BuildOptions, BuildSelector, NonPlatformWheelError,
from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError,
download, get_build_verbosity_extra_flags, get_pip_script,
prepare_command)
@@ -201,6 +202,13 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None:
def build(options: BuildOptions) -> None:
if options.architectures != [Architecture.AMD64, Architecture.x86]:
raise ValueError(textwrap.dedent(f'''
Invalid archs option {options.architectures}. Windows only supports 'amd64,x86' for the
moment. If you want to set emulation architectures on Linux, use CIBW_ARCHS_LINUX
instead.
'''))
temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel'))
built_wheel_dir = temp_dir / 'built_wheel'
repaired_wheel_dir = temp_dir / 'repaired_wheel'