Tests: add --hard-retries option to test runner

- Add the ability to retry by restarting the worker and
  getting a different browser instance, after all
  normal retries have been exhausted. This can sometimes
  be successful when a refresh is not.

Close gh-5438

1 files changed, 8 insertions, 1 deletions

diff --git a/test/runner/run.js b/test/runner/run.js
index 4874fb6f9..0e13e015e 100644
--- a/test/runner/run.js
+++ b/test/runner/run.js
@@ -14,6 +14,7 @@ import { cleanupAllBrowsers, touchBrowser } from "./browserstack/browsers.js";
 import {
 	addBrowserStackRun,
 	getNextBrowserTest,
+	hardRetryTest,
 	retryTest,
 	runAllBrowserStack
 } from "./browserstack/queue.js";
@@ -30,6 +31,7 @@ export async function run( {
 	concurrency,
 	debug,
 	esm,
+	hardRetries,
 	headless,
 	isolate,
 	modules = [],
@@ -72,7 +74,7 @@ export async function run( {
 	// Create the test app and
 	// hook it up to the reporter
 	const reports = Object.create( null );
-	const app = await createTestServer( ( message ) => {
+	const app = await createTestServer( async( message ) => {
 		switch ( message.type ) {
 			case "testEnd": {
 				const reportId = message.id;
@@ -120,6 +122,11 @@ export async function run( {
 					if ( retry ) {
 						return retry;
 					}
+
+					// Return early if hardRetryTest returns true
+					if ( await hardRetryTest( reportId, hardRetries ) ) {
+						return;
+					}
 					errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
 				}