Tests: add --hard-retries option to test runner

- Add the ability to retry by restarting the worker and
  getting a different browser instance, after all
  normal retries have been exhausted. This can sometimes
  be successful when a refresh is not.

Close gh-5438

2 files changed, 53 insertions, 18 deletions

diff --git a/test/runner/browserstack/browsers.js b/test/runner/browserstack/browsers.js
index 3a7da4fc9..218d13fe7 100644
--- a/test/runner/browserstack/browsers.js
+++ b/test/runner/browserstack/browsers.js
@@ -66,7 +66,25 @@ async function waitForAck( worker, { fullBrowser, verbose } ) {
 	} );
 }
 
-async function ensureAcknowledged( worker, restarts ) {
+async function restartWorker( worker ) {
+	await cleanupWorker( worker, worker.options );
+	await createBrowserWorker(
+		worker.url,
+		worker.browser,
+		worker.options,
+		worker.restarts + 1
+	);
+}
+
+export async function restartBrowser( browser ) {
+	const fullBrowser = getBrowserString( browser );
+	const worker = workers[ fullBrowser ];
+	if ( worker ) {
+		await restartWorker( worker );
+	}
+}
+
+async function ensureAcknowledged( worker ) {
 	const fullBrowser = getBrowserString( worker.browser );
 	const verbose = worker.options.verbose;
 	try {
@@ -74,13 +92,7 @@ async function ensureAcknowledged( worker, restarts ) {
 		return worker;
 	} catch ( error ) {
 		console.error( error.message );
-		await cleanupWorker( worker, { verbose } );
-		await createBrowserWorker(
-			worker.url,
-			worker.browser,
-			worker.options,
-			restarts + 1
-		);
+		await restartWorker( worker.browser );
 	}
 }
 
@@ -132,7 +144,7 @@ export async function createBrowserWorker( url, browser, options, restarts = 0 )
 
 	// Wait for the worker to show up in the list
 	// before returning it.
-	return ensureAcknowledged( worker, restarts );
+	return ensureAcknowledged( worker );
 }
 
 export async function setBrowserWorkerUrl( browser, url ) {
@@ -159,13 +171,7 @@ export async function checkLastTouches() {
 					}min.`
 				);
 			}
-			await cleanupWorker( worker, options );
-			await createBrowserWorker(
-				worker.url,
-				worker.browser,
-				options,
-				worker.restarts
-			);
+			await restartWorker( worker );
 		}
 	}
 }
diff --git a/test/runner/browserstack/queue.js b/test/runner/browserstack/queue.js
index c948f29bf..6d1c8d51f 100644
--- a/test/runner/browserstack/queue.js
+++ b/test/runner/browserstack/queue.js
@@ -1,6 +1,11 @@
 import chalk from "chalk";
 import { getBrowserString } from "../lib/getBrowserString.js";
-import { checkLastTouches, createBrowserWorker, setBrowserWorkerUrl } from "./browsers.js";
+import {
+	checkLastTouches,
+	createBrowserWorker,
+	restartBrowser,
+	setBrowserWorkerUrl
+} from "./browsers.js";
 
 const TEST_POLL_TIMEOUT = 1000;
 
@@ -32,6 +37,9 @@ export function getNextBrowserTest( reportId ) {
 }
 
 export function retryTest( reportId, maxRetries ) {
+	if ( !maxRetries ) {
+		return;
+	}
 	const test = queue.find( ( test ) => test.id === reportId );
 	if ( test ) {
 		test.retries++;
@@ -46,10 +54,31 @@ export function retryTest( reportId, maxRetries ) {
 	}
 }
 
+export async function hardRetryTest( reportId, maxHardRetries ) {
+	if ( !maxHardRetries ) {
+		return false;
+	}
+	const test = queue.find( ( test ) => test.id === reportId );
+	if ( test ) {
+		test.hardRetries++;
+		if ( test.hardRetries <= maxHardRetries ) {
+			console.log(
+				`Hard retrying test ${ reportId } for ${ chalk.yellow(
+					test.options.modules.join( ", " )
+				) }...${ test.hardRetries }`
+			);
+			await restartBrowser( test.browser );
+			return true;
+		}
+	}
+	return false;
+}
+
 export function addBrowserStackRun( url, browser, options ) {
 	queue.push( {
 		browser,
 		fullBrowser: getBrowserString( browser ),
+		hardRetries: 0,
 		id: options.reportId,
 		url,
 		options,
@@ -59,7 +88,7 @@ export function addBrowserStackRun( url, browser, options ) {
 }
 
 export async function runAllBrowserStack() {
-	return new Promise( async( resolve, reject )=> {
+	return new Promise( async( resolve, reject ) => {
 		while ( queue.length ) {
 			try {
 				await checkLastTouches();