replace 'pytest-rerunfailures' by 'pytest-retry' to avoid falsy 'fail…

…ure' reporting from successful-retry during coverage for codecov flaky test analysis

Makefile

@@ -381,6 +381,7 @@ ifeq ($(filter $(TEST_VERBOSITY),"--capture"),)
     TEST_VERBOSITY := $(TEST_VERBOSITY) --capture tee-sys
   endif
 endif
+TEST_XARGS ?=
 
 # autogen tests variants with pre-install of dependencies using the '-only' target references
 TESTS := unit func cli workflow online offline no-tb14 spec coverage
@@ -397,56 +398,56 @@ test-all: install-dev test-only		## run all tests (including long running tests)
 .PHONY: test-only
 test-only: mkdir-reports			## run all tests but without prior validation of installed dependencies
 	@echo "Running all tests (including slow and online tests)..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) \
+	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
 		--junitxml "$(REPORTS_DIR)/test-results.xml"'
 
 .PHONY: test-unit-only
 test-unit-only: mkdir-reports 		## run unit tests (skip long running and online tests)
 	@echo "Running unit tests (skip slow and online tests)..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) \
+	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
 		-m "not slow and not online and not functional" --junitxml "$(REPORTS_DIR)/test-results.xml"'
 
 .PHONY: test-func-only
 test-func-only: mkdir-reports   	## run functional tests (online and usage specific)
 	@echo "Running functional tests..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) \
+	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
 		-m "functional" --junitxml "$(REPORTS_DIR)/test-results.xml"'
 
 .PHONY: test-cli-only
 test-cli-only: mkdir-reports   		## run WeaverClient and CLI tests
 	@echo "Running CLI tests..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) \
+	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
 		-m "cli" --junitxml "$(REPORTS_DIR)/test-results.xml"'
 
 .PHONY: test-workflow-only
 test-workflow-only:	mkdir-reports	## run EMS workflow End-2-End tests
 	@echo "Running workflow tests..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) \
+	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
 		-m "workflow" --junitxml "$(REPORTS_DIR)/test-results.xml"'
 
 .PHONY: test-online-only
 test-online-only: mkdir-reports  	## run online tests (running instance required)
 	@echo "Running online tests (running instance required)..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) \
+	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
 		-m "online" --junitxml "$(REPORTS_DIR)/test-results.xml"'
 
 .PHONY: test-offline-only
 test-offline-only: mkdir-reports  	## run offline tests (not marked as online)
 	@echo "Running offline tests (not marked as online)..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) \
+	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
 		-m "not online" --junitxml "$(REPORTS_DIR)/test-results.xml"'
 
 .PHONY: test-no-tb14-only
 test-no-tb14-only: mkdir-reports  	## run all tests except ones marked for 'Testbed-14'
 	@echo "Running all tests except ones marked for 'Testbed-14'..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) \
+	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
 		-m "not testbed14" --junitxml "$(REPORTS_DIR)/test-results.xml"'
 
 .PHONY: test-spec-only
 test-spec-only:	mkdir-reports  ## run tests with custom specification (pytest format) [make SPEC='<spec>' test-spec]
 	@echo "Running custom tests from input specification..."
 	@[ "${SPEC}" ] || ( echo ">> 'SPEC' is not set"; exit 1 )
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) \
+	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
 		-k "${SPEC}" --junitxml "$(REPORTS_DIR)/test-results.xml"'
 
 .PHONY: test-smoke
@@ -459,7 +460,7 @@ test-docker: docker-test    ## alias to 'docker-test' execution smoke test of bu
 test-coverage-only: mkdir-reports  ## run all tests using coverage analysis
 	@echo "Running coverage analysis..."
 	@bash -c '$(CONDA_CMD) coverage run --rcfile="$(APP_ROOT)/setup.cfg" \
-		"$$(which pytest)" "$(APP_ROOT)/tests" --junitxml="$(REPORTS_DIR)/coverage-junit.xml" || true'
+		"$$(which pytest)" "$(APP_ROOT)/tests"  $(TEST_XARGS) --junitxml="$(REPORTS_DIR)/coverage-junit.xml" || true'
 	@bash -c '$(CONDA_CMD) coverage xml --rcfile="$(APP_ROOT)/setup.cfg" -i -o "$(REPORTS_DIR)/coverage.xml"'
 	@bash -c '$(CONDA_CMD) coverage report --rcfile="$(APP_ROOT)/setup.cfg" -i -m'
 	@bash -c '$(CONDA_CMD) coverage html --rcfile="$(APP_ROOT)/setup.cfg" -d "$(REPORTS_DIR)/coverage"'

requirements-dev.txt

@@ -25,7 +25,8 @@ pluggy>=0.7
 pytest<7
 pytest-httpserver>=1.0.7  # support werkzeug>=3
 pytest-server-fixtures
-pytest-rerunfailures
+#pytest-rerunfailures
+pytest-retry
 pydocstyle
 # FIXME: pylint-quotes failing with pylint==3 (https://github.com/edaniszewski/pylint-quotes/issues/29)
 # FIXME: use temporary unofficial version working with pylint>3 (https://github.com/edaniszewski/pylint-quotes/pull/30)

tests/functional/test_cli.py

@@ -562,7 +562,7 @@ def test_execute_inputs_invalid(self):
         ]:
             self.run_execute_inputs_schema_variant(invalid_inputs_schema, expect_success=False)
 
-    @pytest.mark.flaky(reruns=2, reruns_delay=1)
+    @pytest.mark.flaky(retries=2, delay=1)
     def test_execute_manual_monitor_status_and_download_results(self):
         """
         Test a typical case of :term:`Job` execution, result retrieval and download, but with manual monitoring.
@@ -1097,7 +1097,7 @@ def add_docker_pull_ref(cwl, ref):
         cwl["requirements"][CWL_REQUIREMENT_APP_DOCKER] = {"dockerPull": ref}
         return cwl
 
-    @pytest.mark.flaky(reruns=2, reruns_delay=1)
+    @pytest.mark.flaky(retries=2, delay=1)
     def test_deploy_docker_auth_username_password_valid(self):
         """
         Test that username and password arguments can be provided simultaneously for docker login.

tests/functional/test_wps_package.py

@@ -2671,7 +2671,7 @@ def test_execute_with_browsable_directory(self):
             assert all(file.startswith(cwl_stage_dir) for file in output_listing)
             assert all(any(file.endswith(dir_file) for file in output_listing) for dir_file in expect_http_files)
 
-    @pytest.mark.flaky(reruns=2, reruns_delay=1)
+    @pytest.mark.flaky(retries=2, delay=1)
     def test_execute_with_json_listing_directory(self):
         """
         Test that HTTP returning JSON list of directory contents retrieves children files for the process.

tests/processes/test_wps_package.py

@@ -162,7 +162,7 @@ def __init__(self, shell_command, arguments=None, with_message_input=True):
         super(MockProcess, self).__init__(body)
 
 
-@pytest.mark.flaky(reruns=2, reruns_delay=1)
+@pytest.mark.flaky(retries=2, delay=1)
 def test_stdout_stderr_logging_for_commandline_tool_success(caplog):
     """
     Execute a process and assert that stdout is correctly logged to log file upon successful process execution.

tests/test_utils.py

@@ -655,7 +655,7 @@ def mock_sleep(delay):
                 assert all(called == expect for called, expect in zip(sleep_counter["called_with"], intervals))
 
 
-@pytest.mark.flaky(reruns=2, reruns_delay=1)
+@pytest.mark.flaky(retries=2, delay=1)
 def test_request_extra_zero_values():
     """
     Test that zero-value ``retries`` and ``backoff`` are not ignored.