tests: add flaky test handling (#2527)

* chore: add flaky test handling

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* Update test_android.py

---------

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

pyproject.toml

@@ -88,6 +88,7 @@ test = [
     "jinja2",
     "pytest-timeout",
     "pytest-xdist",
+    "pytest-rerunfailures",
     "pytest>=6",
     "setuptools",
     "tomli_w",

test/test_android.py

@@ -90,6 +90,8 @@ def test_android_home(tmp_path, capfd):
     assert "ANDROID_HOME environment variable is not set" in capfd.readouterr().err
 
 
+# Can fail to setup
+@pytest.mark.flaky(reruns=2)
 def test_frontend_good(tmp_path):
     new_c_project().generate(tmp_path)
     wheels = cibuildwheel_run(

test/test_ios.py

@@ -49,7 +49,9 @@ def skip_if_ios_testing_not_supported() -> None:
 # it's easy to overload the CI machine if there are multiple test processes
 # running multithreaded processes. Therefore, they're put in the serial group,
 # which is guaranteed to run single-process.
+# This can also fail the first time sometimes.
 @pytest.mark.serial
+@pytest.mark.flaky(reruns=2)
 @pytest.mark.parametrize(
     "build_config",
     [