chore(ci): speed-up tests (#1752)

* chore(ci): speed-up tests

* back to 2 processes on Azure macOS

.travis.yml

@@ -56,5 +56,5 @@ script: |
   (while true; do echo "travis_keep_alive"; sleep 300; done) &
   SPINNER_PID=$!
   disown
-  $PYTHON ./bin/run_tests.py
+  $PYTHON ./bin/run_tests.py --num-processes 2
   kill -9 ${SPINNER_PID}

azure-pipelines.yml

@@ -24,7 +24,7 @@ jobs:
         versionSpec: '3.8'
     - bash: |
         python -m pip install -e ".[dev]"
-        python ./bin/run_tests.py
+        python ./bin/run_tests.py --num-processes 2
 
 - job: windows_38
   pool: {vmImage: 'windows-2019'}

bin/run_tests.py

@@ -9,10 +9,17 @@ import sys
 from pathlib import Path
 
 if __name__ == "__main__":
+    default_cpu_count = os.cpu_count() or 2
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--run-podman", action="store_true", default=False, help="run podman tests (linux only)"
     )
+    parser.add_argument(
+        "--num-processes",
+        action="store",
+        default=default_cpu_count,
+        help="number of processes to use for testing",
+    )
     args = parser.parse_args()
 
     # move cwd to the project root
@@ -35,7 +42,7 @@ if __name__ == "__main__":
         sys.executable,
         "-m",
         "pytest",
-        "--numprocesses=2",
+        f"--numprocesses={args.num_processes}",
         "-x",
         "--durations",
         "0",

test/test_before_build.py

@@ -12,9 +12,9 @@ project_with_before_build_asserts = test_projects.new_c_project(
         r"""
         import os
 
-        # assert that the Python version as written to pythonversion.txt in the CIBW_BEFORE_BUILD step
+        # assert that the Python version as written to pythonversion_bb.txt in the CIBW_BEFORE_BUILD step
         # is the same one as is currently running.
-        version_file = 'c:\\pythonversion.txt' if sys.platform == 'win32' else '/tmp/pythonversion.txt'
+        version_file = 'c:\\pythonversion_bb.txt' if sys.platform == 'win32' else '/tmp/pythonversion_bb.txt'
         with open(version_file) as f:
             stored_version = f.read()
         print('stored_version', stored_version)
@@ -22,7 +22,7 @@ project_with_before_build_asserts = test_projects.new_c_project(
         assert stored_version == sys.version
 
         # check that the executable also was written
-        executable_file = 'c:\\pythonexecutable.txt' if sys.platform == 'win32' else '/tmp/pythonexecutable.txt'
+        executable_file = 'c:\\pythonexecutable_bb.txt' if sys.platform == 'win32' else '/tmp/pythonexecutable_bb.txt'
         with open(executable_file) as f:
             stored_executable = f.read()
         print('stored_executable', stored_executable)
@@ -44,8 +44,8 @@ def test(tmp_path):
     project_with_before_build_asserts.generate(project_dir)
 
     before_build = (
-        """python -c "import sys; open('{output_dir}pythonversion.txt', 'w').write(sys.version)" && """
+        """python -c "import sys; open('{output_dir}pythonversion_bb.txt', 'w').write(sys.version)" && """
-        '''python -c "import sys; open('{output_dir}pythonexecutable.txt', 'w').write(sys.executable)"'''
+        '''python -c "import sys; open('{output_dir}pythonexecutable_bb.txt', 'w').write(sys.executable)"'''
     )
 
     # build the wheels

test/test_before_test.py

@@ -11,10 +11,10 @@ from unittest import TestCase
 
 class TestBeforeTest(TestCase):
     def test_version(self):
-        # assert that the Python version as written to pythonversion.txt in the CIBW_BEFORE_TEST step
+        # assert that the Python version as written to pythonversion_bt.txt in the CIBW_BEFORE_TEST step
         # is the same one as is currently running.
         # because of use symlinks in MacOS run this test is also need
-        version_file = 'c:\\pythonversion.txt' if sys.platform == 'win32' else '/tmp/pythonversion.txt'
+        version_file = 'c:\\pythonversion_bt.txt' if sys.platform == 'win32' else '/tmp/pythonversion_bt.txt'
         with open(version_file) as f:
             stored_version = f.read()
         print('stored_version', stored_version)
@@ -23,7 +23,7 @@ class TestBeforeTest(TestCase):
 
     def test_prefix(self):
         # check that the prefix also was written
-        prefix_file = 'c:\\pythonprefix.txt' if sys.platform == 'win32' else '/tmp/pythonprefix.txt'
+        prefix_file = 'c:\\pythonprefix_bt.txt' if sys.platform == 'win32' else '/tmp/pythonprefix_bt.txt'
         with open(prefix_file) as f:
             stored_prefix = f.read()
         print('stored_prefix', stored_prefix)
@@ -47,8 +47,8 @@ def test(tmp_path):
         add_env={
             # write python version information to a temporary file, this is
             # checked in setup.py
-            "CIBW_BEFORE_TEST": """python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonprefix.txt', 'w').write(sys.prefix)" && python -m pip install {project}/dependency""",
+            "CIBW_BEFORE_TEST": """python -c "import sys; open('/tmp/pythonversion_bt.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonprefix_bt.txt', 'w').write(sys.prefix)" && python -m pip install {project}/dependency""",
-            "CIBW_BEFORE_TEST_WINDOWS": """python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonprefix.txt', 'w').write(sys.prefix)" && python -m pip install {project}/dependency""",
+            "CIBW_BEFORE_TEST_WINDOWS": """python -c "import sys; open('c:\\pythonversion_bt.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonprefix_bt.txt', 'w').write(sys.prefix)" && python -m pip install {project}/dependency""",
             "CIBW_TEST_REQUIRES": "pytest",
             # the 'false ||' bit is to ensure this command runs in a shell on
             # mac/linux.