Tests: reuse browser workers in BrowserStack tests (#5428)

- reuse BrowserStack workers.
- add support for "latest" and "latest-1" in browser version filters
- add support for specifying non-final browser versions, such as beta versions
- more accurate eslint for files in test/runner
- switched `--no-isolate` command flag to `--isolate`. Now that browser instances are shared, it made more sense to me to default to no isolation unless specified. This turned out to be cleaner because the only place we isolate is in browserstack.yml.
- fixed an issue with retries where it wasn't always waiting for the retried test run
- enable strict mode in test yargs command

1 files changed, 90 insertions, 0 deletions

diff --git a/test/runner/browserstack/queue.js b/test/runner/browserstack/queue.js
new file mode 100644
index 000000000..10ef14a2b
--- /dev/null
+++ b/test/runner/browserstack/queue.js
@@ -0,0 +1,90 @@
+import chalk from "chalk";
+import { getBrowserString } from "../lib/getBrowserString.js";
+import { checkLastTouches, createBrowserWorker, setBrowserWorkerUrl } from "./browsers.js";
+
+const TEST_POLL_TIMEOUT = 1000;
+
+const queue = [];
+
+export function getNextBrowserTest( reportId ) {
+	const index = queue.findIndex( ( test ) => test.id === reportId );
+	if ( index === -1 ) {
+		return;
+	}
+
+	// Remove the completed test from the queue
+	const previousTest = queue[ index ];
+	queue.splice( index, 1 );
+
+	// Find the next test for the same browser
+	for ( const test of queue.slice( index ) ) {
+		if ( test.fullBrowser === previousTest.fullBrowser ) {
+
+			// Set the URL for our tracking
+			setBrowserWorkerUrl( test.browser, test.url );
+			test.running = true;
+
+			// Return the URL for the next test.
+			// listeners.js will use this to set the browser URL.
+			return { url: test.url };
+		}
+	}
+}
+
+export function retryTest( reportId, maxRetries ) {
+	const test = queue.find( ( test ) => test.id === reportId );
+	if ( test ) {
+		test.retries++;
+		if ( test.retries <= maxRetries ) {
+			console.log(
+				`Retrying test ${ reportId } for ${ chalk.yellow(
+					test.options.modules.join( ", " )
+				) }...`
+			);
+			return test;
+		}
+	}
+}
+
+export function addBrowserStackRun( url, browser, options ) {
+	queue.push( {
+		browser,
+		fullBrowser: getBrowserString( browser ),
+		id: options.reportId,
+		url,
+		options,
+		retries: 0,
+		running: false
+	} );
+}
+
+export async function runAllBrowserStack() {
+	return new Promise( async( resolve, reject )=> {
+		while ( queue.length ) {
+			try {
+				await checkLastTouches();
+			} catch ( error ) {
+				reject( error );
+			}
+
+			// Run one test URL per browser at a time
+			const browsersTaken = [];
+			for ( const test of queue ) {
+				if ( browsersTaken.indexOf( test.fullBrowser ) > -1 ) {
+					continue;
+				}
+				browsersTaken.push( test.fullBrowser );
+				if ( !test.running ) {
+					test.running = true;
+					try {
+						await createBrowserWorker( test.url, test.browser, test.options );
+					} catch ( error ) {
+						reject( error );
+					}
+				}
+			}
+			await new Promise( ( resolve ) => setTimeout( resolve, TEST_POLL_TIMEOUT ) );
+		}
+		resolve();
+	} );
+}