Tests: reuse browser workers in BrowserStack tests (#5428)

- reuse BrowserStack workers.
- add support for "latest" and "latest-1" in browser version filters
- add support for specifying non-final browser versions, such as beta versions
- more accurate eslint for files in test/runner
- switched `--no-isolate` command flag to `--isolate`. Now that browser instances are shared, it made more sense to me to default to no isolation unless specified. This turned out to be cleaner because the only place we isolate is in browserstack.yml.
- fixed an issue with retries where it wasn't always waiting for the retried test run
- enable strict mode in test yargs command

1 files changed, 0 insertions, 276 deletions

diff --git a/test/runner/browserstack/workers.js b/test/runner/browserstack/workers.js
deleted file mode 100644
index 8f0ab68f0..000000000
--- a/test/runner/browserstack/workers.js
+++ /dev/null
@@ -1,276 +0,0 @@
-import chalk from "chalk";
-import { getBrowserString } from "../lib/getBrowserString.js";
-import { changeUrl, createWorker, deleteWorker, getWorker } from "./api.js";
-
-const workers = Object.create( null );
-
-// Acknowledge the worker within the time limit.
-// BrowserStack can take much longer spinning up
-// some browsers, such as iOS 15 Safari.
-const ACKNOWLEDGE_WORKER_TIMEOUT = 60 * 1000 * 8;
-const ACKNOWLEDGE_WORKER_INTERVAL = 1000;
-
-// No report after the time limit
-// should refresh the worker
-const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
-const MAX_WORKER_RESTARTS = 5;
-const MAX_WORKER_REFRESHES = 1;
-const POLL_WORKER_TIMEOUT = 1000;
-
-export async function cleanupWorker( reportId, verbose ) {
-	const worker = workers[ reportId ];
-	if ( worker ) {
-		try {
-			delete workers[ reportId ];
-			await deleteWorker( worker.id, verbose );
-		} catch ( error ) {
-			console.error( error );
-		}
-	}
-}
-
-export function debugWorker( reportId ) {
-	const worker = workers[ reportId ];
-	if ( worker ) {
-		worker.debug = true;
-	}
-}
-
-/**
- * Set the last time a request was
- * received related to the worker.
- */
-export function touchWorker( reportId ) {
-	const worker = workers[ reportId ];
-	if ( worker ) {
-		worker.lastTouch = Date.now();
-	}
-}
-
-export function retryTest( reportId, retries ) {
-	const worker = workers[ reportId ];
-	if ( worker ) {
-		worker.retries ||= 0;
-		worker.retries++;
-		if ( worker.retries <= retries ) {
-			worker.retry = true;
-			console.log( `\nRetrying test ${ reportId }...${ worker.retries }` );
-			return true;
-		}
-	}
-	return false;
-}
-
-export async function cleanupAllWorkers( verbose ) {
-	const workersRemaining = Object.keys( workers ).length;
-	if ( workersRemaining ) {
-		if ( verbose ) {
-			console.log(
-				`Stopping ${ workersRemaining } stray worker${
-					workersRemaining > 1 ? "s" : ""
-				}...`
-			);
-		}
-		await Promise.all(
-			Object.values( workers ).map( ( worker ) => deleteWorker( worker.id, verbose ) )
-		);
-	}
-}
-
-async function waitForAck( id, verbose ) {
-	return new Promise( ( resolve, reject ) => {
-		const interval = setInterval( () => {
-			const worker = workers[ id ];
-			if ( !worker ) {
-				clearTimeout( timeout );
-				clearInterval( interval );
-				return reject( new Error( `Worker ${ id } not found.` ) );
-			}
-			if ( worker.lastTouch ) {
-				if ( verbose ) {
-					console.log( `\nWorker ${ id } acknowledged.` );
-				}
-				clearTimeout( timeout );
-				clearInterval( interval );
-				resolve();
-			}
-		}, ACKNOWLEDGE_WORKER_INTERVAL );
-		const timeout = setTimeout( () => {
-			clearInterval( interval );
-			const worker = workers[ id ];
-			reject(
-				new Error(
-					`Worker ${
-						worker ? worker.id : ""
-					} for test ${ id } not acknowledged after ${
-						ACKNOWLEDGE_WORKER_TIMEOUT / 1000
-					}s.`
-				)
-			);
-		}, ACKNOWLEDGE_WORKER_TIMEOUT );
-	} );
-}
-
-export async function runWorker(
-	url,
-	browser,
-	options,
-	restarts = 0
-) {
-	const { modules, reportId, runId, verbose } = options;
-	const worker = await createWorker( {
-		...browser,
-		url: encodeURI( url ),
-		project: "jquery",
-		build: `Run ${ runId }`,
-		name: `${ modules.join( "," ) } (${ reportId })`,
-
-		// Set the max here, so that we can
-		// control the timeout
-		timeout: 1800,
-
-		// Not documented in the API docs,
-		// but required to make local testing work.
-		// See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
-		"browserstack.local": true,
-		"browserstack.localIdentifier": runId
-	} );
-
-	workers[ reportId ] = worker;
-
-	const timeMessage = `\nWorker ${
-		worker.id
-	} created for test ${ reportId } (${ chalk.yellow( getBrowserString( browser ) ) })`;
-
-	if ( verbose ) {
-		console.time( timeMessage );
-	}
-
-	async function retryWorker() {
-		await cleanupWorker( reportId, verbose );
-		if ( verbose ) {
-			console.log( `Retrying worker for test ${ reportId }...${ restarts + 1 }` );
-		}
-		return runWorker( url, browser, options, restarts + 1 );
-	}
-
-	// Wait for the worker to be acknowledged
-	try {
-		await waitForAck( reportId );
-	} catch ( error ) {
-		if ( !workers[ reportId ] ) {
-
-			// The worker has already been cleaned up
-			return;
-		}
-
-		if ( restarts < MAX_WORKER_RESTARTS ) {
-			return retryWorker();
-		}
-
-		throw error;
-	}
-
-	if ( verbose ) {
-		console.timeEnd( timeMessage );
-	}
-
-	let refreshes = 0;
-	let loggedStarted = false;
-	return new Promise( ( resolve, reject ) => {
-		async function refreshWorker() {
-			try {
-				await changeUrl( worker.id, url );
-				touchWorker( reportId );
-				return tick();
-			} catch ( error ) {
-				if ( !workers[ reportId ] ) {
-
-					// The worker has already been cleaned up
-					return resolve();
-				}
-				console.error( error );
-				return retryWorker().then( resolve, reject );
-			}
-		}
-
-		async function checkWorker() {
-			const worker = workers[ reportId ];
-
-			if ( !worker || worker.debug ) {
-				return resolve();
-			}
-
-			let fetchedWorker;
-			try {
-				fetchedWorker = await getWorker( worker.id );
-			} catch ( error ) {
-				return reject( error );
-			}
-			if (
-				!fetchedWorker ||
-				( fetchedWorker.status !== "running" && fetchedWorker.status !== "queue" )
-			) {
-				return resolve();
-			}
-
-			if ( verbose && !loggedStarted ) {
-				loggedStarted = true;
-				console.log(
-					`\nTest ${ chalk.bold( reportId ) } is ${
-						worker.status === "running" ? "running" : "in the queue"
-					}.`
-				);
-				console.log( `  View at ${ fetchedWorker.browser_url }.` );
-			}
-
-			// Refresh the worker if a retry is triggered
-			if ( worker.retry ) {
-				worker.retry = false;
-
-				// Reset recovery refreshes
-				refreshes = 0;
-				return refreshWorker();
-			}
-
-			if ( worker.lastTouch > Date.now() - RUN_WORKER_TIMEOUT ) {
-				return tick();
-			}
-
-			refreshes++;
-
-			if ( refreshes >= MAX_WORKER_REFRESHES ) {
-				if ( restarts < MAX_WORKER_RESTARTS ) {
-					if ( verbose ) {
-						console.log(
-							`Worker ${ worker.id } not acknowledged after ${
-								ACKNOWLEDGE_WORKER_TIMEOUT / 1000
-							}s.`
-						);
-					}
-					return retryWorker().then( resolve, reject );
-				}
-				await cleanupWorker( reportId, verbose );
-				return reject(
-					new Error(
-						`Worker ${ worker.id } for test ${ reportId } timed out after ${ MAX_WORKER_RESTARTS } restarts.`
-					)
-				);
-			}
-
-			if ( verbose ) {
-				console.log(
-					`\nRefreshing worker ${ worker.id } for test ${ reportId }...${ refreshes }`
-				);
-			}
-
-			return refreshWorker();
-		}
-
-		function tick() {
-			setTimeout( checkWorker, POLL_WORKER_TIMEOUT );
-		}
-
-		checkWorker();
-	} );
-}