Tests: reuse browser workers in BrowserStack tests (#5428)

- reuse BrowserStack workers.
- add support for "latest" and "latest-1" in browser version filters
- add support for specifying non-final browser versions, such as beta versions
- more accurate eslint for files in test/runner
- switched `--no-isolate` command flag to `--isolate`. Now that browser instances are shared, it made more sense to me to default to no isolation unless specified. This turned out to be cleaner because the only place we isolate is in browserstack.yml.
- fixed an issue with retries where it wasn't always waiting for the retried test run
- enable strict mode in test yargs command

3 files changed, 72 insertions, 2 deletions

diff --git a/test/runner/selenium/createDriver.js b/test/runner/selenium/createDriver.js
index 765ebb847..d1680b22d 100644
--- a/test/runner/selenium/createDriver.js
+++ b/test/runner/selenium/createDriver.js
@@ -55,7 +55,8 @@ export default async function createDriver( { browserName, headless, verbose } )
 		edgeOptions.addArguments( "--headless=new" );
 		if ( !browserSupportsHeadless( browserName ) ) {
 			console.log(
-				`Headless mode is not supported for ${ browserName }. Running in normal mode instead.`
+				`Headless mode is not supported for ${ browserName }.` +
+					"Running in normal mode instead."
 			);
 		}
 	}
diff --git a/test/runner/selenium/queue.js b/test/runner/selenium/queue.js
new file mode 100644
index 000000000..863db4d9b
--- /dev/null
+++ b/test/runner/selenium/queue.js
@@ -0,0 +1,70 @@
+// Build a queue that runs both browsers and modules
+// in parallel when the length reaches the concurrency limit
+// and refills the queue when one promise resolves.
+
+import chalk from "chalk";
+import { getBrowserString } from "../lib/getBrowserString.js";
+import { runSelenium } from "./runSelenium.js";
+import { runJSDOM } from "../jsdom.js";
+
+const promises = [];
+const queue = [];
+
+const SELENIUM_WAIT_TIME = 100;
+
+// Limit concurrency to 8 by default in selenium
+// BrowserStack defaults to the max allowed by the plan
+// More than this will log MaxListenersExceededWarning
+const MAX_CONCURRENCY = 8;
+
+export function addSeleniumRun( url, browser, options ) {
+	queue.push( { url, browser, options } );
+}
+
+export async function runAllSelenium( { concurrency = MAX_CONCURRENCY, verbose } ) {
+	while ( queue.length ) {
+		const next = queue.shift();
+		const { url, browser, options } = next;
+
+		const fullBrowser = getBrowserString( browser, options.headless );
+		console.log(
+			`\nRunning ${ chalk.yellow( options.modules.join( ", " ) ) } tests ` +
+				`in ${ chalk.yellow( fullBrowser ) } (${ chalk.bold( options.reportId ) })...`
+		);
+
+		// Wait enough time between requests
+		// to give concurrency a chance to update.
+		// In selenium, this helps avoid undici connect timeout errors.
+		await new Promise( ( resolve ) => setTimeout( resolve, SELENIUM_WAIT_TIME ) );
+
+		if ( verbose ) {
+			console.log( `\nTests remaining: ${ queue.length + 1 }.` );
+		}
+
+		let promise;
+		if ( browser.browser === "jsdom" ) {
+			promise = runJSDOM( url, options );
+		} else {
+			promise = runSelenium( url, browser, options );
+		}
+
+		// Remove the promise from the list when it resolves
+		promise.then( () => {
+			const index = promises.indexOf( promise );
+			if ( index !== -1 ) {
+				promises.splice( index, 1 );
+			}
+		} );
+
+		// Add the promise to the list
+		promises.push( promise );
+
+		// Wait until at least one promise resolves
+		// if we've reached the concurrency limit
+		if ( promises.length >= concurrency ) {
+			await Promise.any( promises );
+		}
+	}
+
+	await Promise.all( promises );
+}
diff --git a/test/runner/selenium/runSelenium.js b/test/runner/selenium/runSelenium.js
index 247cd8472..848db36c7 100644
--- a/test/runner/selenium/runSelenium.js
+++ b/test/runner/selenium/runSelenium.js
@@ -1,4 +1,3 @@
-import chalk from "chalk";
 import createDriver from "./createDriver.js";
 
 export async function runSelenium(