Tests: reuse browser workers in BrowserStack tests (#5428)

- reuse BrowserStack workers.
- add support for "latest" and "latest-1" in browser version filters
- add support for specifying non-final browser versions, such as beta versions
- more accurate eslint for files in test/runner
- switched `--no-isolate` command flag to `--isolate`. Now that browser instances are shared, it made more sense to me to default to no isolation unless specified. This turned out to be cleaner because the only place we isolate is in browserstack.yml.
- fixed an issue with retries where it wasn't always waiting for the retried test run
- enable strict mode in test yargs command

1 files changed, 76 insertions, 33 deletions

diff --git a/test/runner/browserstack/api.js b/test/runner/browserstack/api.js
index 40982c9b3..632f90c3b 100644
--- a/test/runner/browserstack/api.js
+++ b/test/runner/browserstack/api.js
@@ -14,6 +14,7 @@ const accessKey = process.env.BROWSERSTACK_ACCESS_KEY;
 // iOS has null for version numbers,
 // and we do not need a similar check for OS versions.
 const rfinalVersion = /(?:^[0-9\.]+$)|(?:^null$)/;
+const rlatest = /^latest-(\d+)$/;
 
 const rnonDigits = /(?:[^\d\.]+)|(?:20\d{2})/g;
 
@@ -84,6 +85,15 @@ function compareVersionNumbers( a, b ) {
 			return 1;
 		}
 	}
+
+	if ( rnonDigits.test( a ) && !rnonDigits.test( b ) ) {
+		return -1;
+	}
+	if ( !rnonDigits.test( a ) && rnonDigits.test( b ) ) {
+		return 1;
+	}
+
+	return 0;
 }
 
 function sortBrowsers( a, b ) {
@@ -148,17 +158,62 @@ export async function filterBrowsers( filter ) {
 	const filterOsVersion = ( filter.os_version ?? "" ).toLowerCase();
 	const filterDevice = ( filter.device ?? "" ).toLowerCase();
 
-	return browsers.filter( ( browser ) => {
+	const filteredWithoutVersion = browsers.filter( ( browser ) => {
 		return (
 			( !filterBrowser || filterBrowser === browser.browser.toLowerCase() ) &&
-			( !filterVersion ||
-				matchVersion( browser.browser_version, filterVersion ) ) &&
 			( !filterOs || filterOs === browser.os.toLowerCase() ) &&
-			( !filterOsVersion ||
-				filterOsVersion === browser.os_version.toLowerCase() ) &&
+			( !filterOsVersion || matchVersion( browser.os_version, filterOsVersion ) ) &&
 			( !filterDevice || filterDevice === ( browser.device || "" ).toLowerCase() )
 		);
 	} );
+
+	if ( !filterVersion ) {
+		return filteredWithoutVersion;
+	}
+
+	if ( filterVersion.startsWith( "latest" ) ) {
+		const groupedByName = filteredWithoutVersion
+			.filter( ( b ) => rfinalVersion.test( b.browser_version ) )
+			.reduce( ( acc, browser ) => {
+				acc[ browser.browser ] = acc[ browser.browser ] ?? [];
+				acc[ browser.browser ].push( browser );
+				return acc;
+			}, Object.create( null ) );
+
+		const filtered = [];
+		for ( const group of Object.values( groupedByName ) ) {
+			const latest = group[ group.length - 1 ];
+
+			// Mobile devices do not have browser version.
+			// Skip the version check for these,
+			// but include the latest in the list if it made it
+			// through filtering.
+			if ( !latest.browser_version ) {
+
+				// Do not include in the list for latest-n.
+				if ( filterVersion === "latest" ) {
+					filtered.push( latest );
+				}
+				continue;
+			}
+
+			// Get the latest version and subtract the number from the filter,
+			// ignoring any patch versions, which may differ between major versions.
+			const num = rlatest.exec( filterVersion );
+			const version = parseInt( latest.browser_version ) - ( num ? num[ 1 ] : 0 );
+			const match = group.findLast( ( browser ) => {
+				return matchVersion( browser.browser_version, version.toString() );
+			} );
+			if ( match ) {
+				filtered.push( match );
+			}
+		}
+		return filtered;
+	}
+
+	return filteredWithoutVersion.filter( ( browser ) => {
+		return matchVersion( browser.browser_version, filterVersion );
+	} );
 }
 
 export async function listBrowsers( filter ) {
@@ -177,13 +232,11 @@ export async function listBrowsers( filter ) {
 }
 
 export async function getLatestBrowser( filter ) {
+	if ( !filter.browser_version ) {
+		filter.browser_version = "latest";
+	}
 	const browsers = await filterBrowsers( filter );
-
-	// The list is sorted in ascending order,
-	// so the last item is the latest.
-	return browsers.findLast( ( browser ) =>
-		rfinalVersion.test( browser.browser_version )
-	);
+	return browsers[ browsers.length - 1 ];
 }
 
 /**
@@ -229,11 +282,8 @@ export function getWorker( id ) {
 	return fetchAPI( `/worker/${ id }` );
 }
 
-export async function deleteWorker( id, verbose ) {
-	await fetchAPI( `/worker/${ id }`, { method: "DELETE" } );
-	if ( verbose ) {
-		console.log( `\nWorker ${ id } stopped.` );
-	}
+export async function deleteWorker( id ) {
+	return fetchAPI( `/worker/${ id }`, { method: "DELETE" } );
 }
 
 export function getWorkers() {
@@ -241,20 +291,6 @@ export function getWorkers() {
 }
 
 /**
- * Change the URL of a worker,
- * or refresh if it's the same URL.
- */
-export function changeUrl( id, url ) {
-	return fetchAPI( `/worker/${ id }/url.json`, {
-		method: "PUT",
-		body: JSON.stringify( {
-			timeout: 20,
-			url: encodeURI( url )
-		} )
-	} );
-}
-
-/**
  * Stop all workers
  */
 export async function stopWorkers() {
@@ -262,15 +298,17 @@ export async function stopWorkers() {
 
 	// Run each request on its own
 	// to avoid connect timeout errors.
+	console.log( `${ workers.length } workers running...` );
 	for ( const worker of workers ) {
 		try {
-			await deleteWorker( worker.id, true );
+			await deleteWorker( worker.id );
 		} catch ( error ) {
 
 			// Log the error, but continue trying to remove workers.
 			console.error( error );
 		}
 	}
+	console.log( "All workers stopped." );
 }
 
 /**
@@ -284,6 +322,11 @@ export function getPlan() {
 }
 
 export async function getAvailableSessions() {
-	const [ plan, workers ] = await Promise.all( [ getPlan(), getWorkers() ] );
-	return plan.parallel_sessions_max_allowed - workers.length;
+	try {
+		const [ plan, workers ] = await Promise.all( [ getPlan(), getWorkers() ] );
+		return plan.parallel_sessions_max_allowed - workers.length;
+	} catch ( error ) {
+		console.error( error );
+		return 0;
+	}
 }