5 files changed, 616 insertions, 0 deletions
diff --git a/test/runner/browserstack/api.js b/test/runner/browserstack/api.js
new file mode 100644
index 000000000..40982c9b3
--- /dev/null
+++ b/test/runner/browserstack/api.js
@@ -0,0 +1,289 @@
+/**
+ * Browserstack API is documented at
+ * https://github.com/browserstack/api
+ */
+
+import { createAuthHeader } from "./createAuthHeader.js";
+
+const browserstackApi = "https://api.browserstack.com";
+const apiVersion = 5;
+
+const username = process.env.BROWSERSTACK_USERNAME;
+const accessKey = process.env.BROWSERSTACK_ACCESS_KEY;
+
+// iOS has null for version numbers,
+// and we do not need a similar check for OS versions.
+const rfinalVersion = /(?:^[0-9\.]+$)|(?:^null$)/;
+
+const rnonDigits = /(?:[^\d\.]+)|(?:20\d{2})/g;
+
+async function fetchAPI( path, options = {}, versioned = true ) {
+	if ( !username || !accessKey ) {
+		throw new Error(
+			"BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables must be set."
+		);
+	}
+	const init = {
+		method: "GET",
+		...options,
+		headers: {
+			authorization: createAuthHeader( username, accessKey ),
+			accept: "application/json",
+			"content-type": "application/json",
+			...options.headers
+		}
+	};
+	const response = await fetch(
+		`${ browserstackApi }/${ versioned ? `${ apiVersion }/` : "" }${ path }`,
+		init
+	);
+	if ( !response.ok ) {
+		console.log(
+			`\n${ init.method } ${ path }`,
+			response.status,
+			response.statusText
+		);
+		throw new Error( `Error fetching ${ path }` );
+	}
+	return response.json();
+}
+
+/**
+ * =============================
+ * Browsers API
+ * =============================
+ */
+
+function compareVersionNumbers( a, b ) {
+	if ( a != null && b == null ) {
+		return -1;
+	}
+	if ( a == null && b != null ) {
+		return 1;
+	}
+	if ( a == null && b == null ) {
+		return 0;
+	}
+	const aParts = a.replace( rnonDigits, "" ).split( "." );
+	const bParts = b.replace( rnonDigits, "" ).split( "." );
+
+	if ( aParts.length > bParts.length ) {
+		return -1;
+	}
+	if ( aParts.length < bParts.length ) {
+		return 1;
+	}
+
+	for ( let i = 0; i < aParts.length; i++ ) {
+		const aPart = Number( aParts[ i ] );
+		const bPart = Number( bParts[ i ] );
+		if ( aPart < bPart ) {
+			return -1;
+		}
+		if ( aPart > bPart ) {
+			return 1;
+		}
+	}
+}
+
+function sortBrowsers( a, b ) {
+	if ( a.browser < b.browser ) {
+		return -1;
+	}
+	if ( a.browser > b.browser ) {
+		return 1;
+	}
+	const browserComparison = compareVersionNumbers(
+		a.browser_version,
+		b.browser_version
+	);
+	if ( browserComparison ) {
+		return browserComparison;
+	}
+	if ( a.os < b.os ) {
+		return -1;
+	}
+	if ( a.os > b.os ) {
+		return 1;
+	}
+	const osComparison = compareVersionNumbers( a.os_version, b.os_version );
+	if ( osComparison ) {
+		return osComparison;
+	}
+	const deviceComparison = compareVersionNumbers( a.device, b.device );
+	if ( deviceComparison ) {
+		return deviceComparison;
+	}
+	return 0;
+}
+
+export async function getBrowsers( { flat = false } = {} ) {
+	const query = new URLSearchParams();
+	if ( flat ) {
+		query.append( "flat", true );
+	}
+	const browsers = await fetchAPI( `/browsers?${ query }` );
+	return browsers.sort( sortBrowsers );
+}
+
+function matchVersion( browserVersion, version ) {
+	if ( !version ) {
+		return false;
+	}
+	const regex = new RegExp(
+		`^${ version.replace( /\\/g, "\\\\" ).replace( /\./g, "\\." ) }\\b`,
+		"i"
+	);
+	return regex.test( browserVersion );
+}
+
+export async function filterBrowsers( filter ) {
+	const browsers = await getBrowsers( { flat: true } );
+	if ( !filter ) {
+		return browsers;
+	}
+	const filterBrowser = ( filter.browser ?? "" ).toLowerCase();
+	const filterVersion = ( filter.browser_version ?? "" ).toLowerCase();
+	const filterOs = ( filter.os ?? "" ).toLowerCase();
+	const filterOsVersion = ( filter.os_version ?? "" ).toLowerCase();
+	const filterDevice = ( filter.device ?? "" ).toLowerCase();
+
+	return browsers.filter( ( browser ) => {
+		return (
+			( !filterBrowser || filterBrowser === browser.browser.toLowerCase() ) &&
+			( !filterVersion ||
+				matchVersion( browser.browser_version, filterVersion ) ) &&
+			( !filterOs || filterOs === browser.os.toLowerCase() ) &&
+			( !filterOsVersion ||
+				filterOsVersion === browser.os_version.toLowerCase() ) &&
+			( !filterDevice || filterDevice === ( browser.device || "" ).toLowerCase() )
+		);
+	} );
+}
+
+export async function listBrowsers( filter ) {
+	const browsers = await filterBrowsers( filter );
+	console.log( "Available browsers:" );
+	for ( const browser of browsers ) {
+		let message = `    ${ browser.browser }_`;
+		if ( browser.device ) {
+			message += `:${ browser.device }_`;
+		} else {
+			message += `${ browser.browser_version }_`;
+		}
+		message += `${ browser.os }_${ browser.os_version }`;
+		console.log( message );
+	}
+}
+
+export async function getLatestBrowser( filter ) {
+	const browsers = await filterBrowsers( filter );
+
+	// The list is sorted in ascending order,
+	// so the last item is the latest.
+	return browsers.findLast( ( browser ) =>
+		rfinalVersion.test( browser.browser_version )
+	);
+}
+
+/**
+ * =============================
+ * Workers API
+ * =============================
+ */
+
+/**
+ * A browser object may only have one of `browser` or `device` set;
+ * which property is set will depend on `os`.
+ *
+ * `options`: is an object with the following properties:
+ *   `os`: The operating system.
+ *   `os_version`: The operating system version.
+ *   `browser`: The browser name.
+ *   `browser_version`: The browser version.
+ *   `device`: The device name.
+ *   `url` (optional): Which URL to navigate to upon creation.
+ *   `timeout` (optional): Maximum life of the worker (in seconds). Maximum value of `1800`. Specifying `0` will use the default of `300`.
+ *   `name` (optional): Provide a name for the worker.
+ *   `build` (optional): Group workers into a build.
+ *   `project` (optional): Provide the project the worker belongs to.
+ *   `resolution` (optional): Specify the screen resolution (e.g. "1024x768").
+ *   `browserstack.local` (optional): Set to `true` to mark as local testing.
+ *   `browserstack.video` (optional): Set to `false` to disable video recording.
+ *   `browserstack.localIdentifier` (optional): ID of the local tunnel.
+ */
+export function createWorker( options ) {
+	return fetchAPI( "/worker", {
+		method: "POST",
+		body: JSON.stringify( options )
+	} );
+}
+
+/**
+ * Returns a worker object, if one exists, with the following properties:
+ *   `id`: The worker id.
+ *   `status`: A string representing the current status of the worker.
+ *     Possible statuses: `"running"`, `"queue"`.
+ */
+export function getWorker( id ) {
+	return fetchAPI( `/worker/${ id }` );
+}
+
+export async function deleteWorker( id, verbose ) {
+	await fetchAPI( `/worker/${ id }`, { method: "DELETE" } );
+	if ( verbose ) {
+		console.log( `\nWorker ${ id } stopped.` );
+	}
+}
+
+export function getWorkers() {
+	return fetchAPI( "/workers" );
+}
+
+/**
+ * Change the URL of a worker,
+ * or refresh if it's the same URL.
+ */
+export function changeUrl( id, url ) {
+	return fetchAPI( `/worker/${ id }/url.json`, {
+		method: "PUT",
+		body: JSON.stringify( {
+			timeout: 20,
+			url: encodeURI( url )
+		} )
+	} );
+}
+
+/**
+ * Stop all workers
+ */
+export async function stopWorkers() {
+	const workers = await getWorkers();
+
+	// Run each request on its own
+	// to avoid connect timeout errors.
+	for ( const worker of workers ) {
+		try {
+			await deleteWorker( worker.id, true );
+		} catch ( error ) {
+
+			// Log the error, but continue trying to remove workers.
+			console.error( error );
+		}
+	}
+}
+
+/**
+ * =============================
+ * Plan API
+ * =============================
+ */
+
+export function getPlan() {
+	return fetchAPI( "/automate/plan.json", {}, false );
+}
+
+export async function getAvailableSessions() {
+	const [ plan, workers ] = await Promise.all( [ getPlan(), getWorkers() ] );
+	return plan.parallel_sessions_max_allowed - workers.length;
+}
diff --git a/test/runner/browserstack/buildBrowserFromString.js b/test/runner/browserstack/buildBrowserFromString.js
new file mode 100644
index 000000000..55aa38053
--- /dev/null
+++ b/test/runner/browserstack/buildBrowserFromString.js
@@ -0,0 +1,10 @@
+export function buildBrowserFromString( str ) {
+	const [ browser, versionOrDevice, os, osVersion ] = str.split( "_" );
+
+	// If the version starts with a colon, it's a device
+	if ( versionOrDevice && versionOrDevice.startsWith( ":" ) ) {
+		return { browser, device: versionOrDevice.slice( 1 ), os, os_version: osVersion };
+	}
+
+	return { browser, browser_version: versionOrDevice, os, os_version: osVersion };
+}
diff --git a/test/runner/browserstack/createAuthHeader.js b/test/runner/browserstack/createAuthHeader.js
new file mode 100644
index 000000000..fe4831e9a
--- /dev/null
+++ b/test/runner/browserstack/createAuthHeader.js
@@ -0,0 +1,7 @@
+const textEncoder = new TextEncoder();
+
+export function createAuthHeader( username, accessKey ) {
+	const encoded = textEncoder.encode( `${ username }:${ accessKey }` );
+	const base64 = btoa( String.fromCodePoint.apply( null, encoded ) );
+	return `Basic ${ base64 }`;
+}
diff --git a/test/runner/browserstack/local.js b/test/runner/browserstack/local.js
new file mode 100644
index 000000000..c84cf155c
--- /dev/null
+++ b/test/runner/browserstack/local.js
@@ -0,0 +1,34 @@
+import browserstackLocal from "browserstack-local";
+
+export async function localTunnel( localIdentifier, opts = {} ) {
+	const tunnel = new browserstackLocal.Local();
+
+	return new Promise( ( resolve, reject ) => {
+
+		// https://www.browserstack.com/docs/local-testing/binary-params
+		tunnel.start(
+			{
+				"enable-logging-for-api": "",
+				localIdentifier,
+				...opts
+			},
+			async( error ) => {
+				if ( error || !tunnel.isRunning() ) {
+					return reject( error );
+				}
+				resolve( {
+					stop: function stopTunnel() {
+						return new Promise( ( resolve, reject ) => {
+							tunnel.stop( ( error ) => {
+								if ( error ) {
+									return reject( error );
+								}
+								resolve();
+							} );
+						} );
+					}
+				} );
+			}
+		);
+	} );
+}
diff --git a/test/runner/browserstack/workers.js b/test/runner/browserstack/workers.js
new file mode 100644
index 000000000..8f0ab68f0
--- /dev/null
+++ b/test/runner/browserstack/workers.js
@@ -0,0 +1,276 @@
+import chalk from "chalk";
+import { getBrowserString } from "../lib/getBrowserString.js";
+import { changeUrl, createWorker, deleteWorker, getWorker } from "./api.js";
+
+const workers = Object.create( null );
+
+// Acknowledge the worker within the time limit.
+// BrowserStack can take much longer spinning up
+// some browsers, such as iOS 15 Safari.
+const ACKNOWLEDGE_WORKER_TIMEOUT = 60 * 1000 * 8;
+const ACKNOWLEDGE_WORKER_INTERVAL = 1000;
+
+// No report after the time limit
+// should refresh the worker
+const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
+const MAX_WORKER_RESTARTS = 5;
+const MAX_WORKER_REFRESHES = 1;
+const POLL_WORKER_TIMEOUT = 1000;
+
+export async function cleanupWorker( reportId, verbose ) {
+	const worker = workers[ reportId ];
+	if ( worker ) {
+		try {
+			delete workers[ reportId ];
+			await deleteWorker( worker.id, verbose );
+		} catch ( error ) {
+			console.error( error );
+		}
+	}
+}
+
+export function debugWorker( reportId ) {
+	const worker = workers[ reportId ];
+	if ( worker ) {
+		worker.debug = true;
+	}
+}
+
+/**
+ * Set the last time a request was
+ * received related to the worker.
+ */
+export function touchWorker( reportId ) {
+	const worker = workers[ reportId ];
+	if ( worker ) {
+		worker.lastTouch = Date.now();
+	}
+}
+
+export function retryTest( reportId, retries ) {
+	const worker = workers[ reportId ];
+	if ( worker ) {
+		worker.retries ||= 0;
+		worker.retries++;
+		if ( worker.retries <= retries ) {
+			worker.retry = true;
+			console.log( `\nRetrying test ${ reportId }...${ worker.retries }` );
+			return true;
+		}
+	}
+	return false;
+}
+
+export async function cleanupAllWorkers( verbose ) {
+	const workersRemaining = Object.keys( workers ).length;
+	if ( workersRemaining ) {
+		if ( verbose ) {
+			console.log(
+				`Stopping ${ workersRemaining } stray worker${
+					workersRemaining > 1 ? "s" : ""
+				}...`
+			);
+		}
+		await Promise.all(
+			Object.values( workers ).map( ( worker ) => deleteWorker( worker.id, verbose ) )
+		);
+	}
+}
+
+async function waitForAck( id, verbose ) {
+	return new Promise( ( resolve, reject ) => {
+		const interval = setInterval( () => {
+			const worker = workers[ id ];
+			if ( !worker ) {
+				clearTimeout( timeout );
+				clearInterval( interval );
+				return reject( new Error( `Worker ${ id } not found.` ) );
+			}
+			if ( worker.lastTouch ) {
+				if ( verbose ) {
+					console.log( `\nWorker ${ id } acknowledged.` );
+				}
+				clearTimeout( timeout );
+				clearInterval( interval );
+				resolve();
+			}
+		}, ACKNOWLEDGE_WORKER_INTERVAL );
+		const timeout = setTimeout( () => {
+			clearInterval( interval );
+			const worker = workers[ id ];
+			reject(
+				new Error(
+					`Worker ${
+						worker ? worker.id : ""
+					} for test ${ id } not acknowledged after ${
+						ACKNOWLEDGE_WORKER_TIMEOUT / 1000
+					}s.`
+				)
+			);
+		}, ACKNOWLEDGE_WORKER_TIMEOUT );
+	} );
+}
+
+export async function runWorker(
+	url,
+	browser,
+	options,
+	restarts = 0
+) {
+	const { modules, reportId, runId, verbose } = options;
+	const worker = await createWorker( {
+		...browser,
+		url: encodeURI( url ),
+		project: "jquery",
+		build: `Run ${ runId }`,
+		name: `${ modules.join( "," ) } (${ reportId })`,
+
+		// Set the max here, so that we can
+		// control the timeout
+		timeout: 1800,
+
+		// Not documented in the API docs,
+		// but required to make local testing work.
+		// See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
+		"browserstack.local": true,
+		"browserstack.localIdentifier": runId
+	} );
+
+	workers[ reportId ] = worker;
+
+	const timeMessage = `\nWorker ${
+		worker.id
+	} created for test ${ reportId } (${ chalk.yellow( getBrowserString( browser ) ) })`;
+
+	if ( verbose ) {
+		console.time( timeMessage );
+	}
+
+	async function retryWorker() {
+		await cleanupWorker( reportId, verbose );
+		if ( verbose ) {
+			console.log( `Retrying worker for test ${ reportId }...${ restarts + 1 }` );
+		}
+		return runWorker( url, browser, options, restarts + 1 );
+	}
+
+	// Wait for the worker to be acknowledged
+	try {
+		await waitForAck( reportId );
+	} catch ( error ) {
+		if ( !workers[ reportId ] ) {
+
+			// The worker has already been cleaned up
+			return;
+		}
+
+		if ( restarts < MAX_WORKER_RESTARTS ) {
+			return retryWorker();
+		}
+
+		throw error;
+	}
+
+	if ( verbose ) {
+		console.timeEnd( timeMessage );
+	}
+
+	let refreshes = 0;
+	let loggedStarted = false;
+	return new Promise( ( resolve, reject ) => {
+		async function refreshWorker() {
+			try {
+				await changeUrl( worker.id, url );
+				touchWorker( reportId );
+				return tick();
+			} catch ( error ) {
+				if ( !workers[ reportId ] ) {
+
+					// The worker has already been cleaned up
+					return resolve();
+				}
+				console.error( error );
+				return retryWorker().then( resolve, reject );
+			}
+		}
+
+		async function checkWorker() {
+			const worker = workers[ reportId ];
+
+			if ( !worker || worker.debug ) {
+				return resolve();
+			}
+
+			let fetchedWorker;
+			try {
+				fetchedWorker = await getWorker( worker.id );
+			} catch ( error ) {
+				return reject( error );
+			}
+			if (
+				!fetchedWorker ||
+				( fetchedWorker.status !== "running" && fetchedWorker.status !== "queue" )
+			) {
+				return resolve();
+			}
+
+			if ( verbose && !loggedStarted ) {
+				loggedStarted = true;
+				console.log(
+					`\nTest ${ chalk.bold( reportId ) } is ${
+						worker.status === "running" ? "running" : "in the queue"
+					}.`
+				);
+				console.log( `  View at ${ fetchedWorker.browser_url }.` );
+			}
+
+			// Refresh the worker if a retry is triggered
+			if ( worker.retry ) {
+				worker.retry = false;
+
+				// Reset recovery refreshes
+				refreshes = 0;
+				return refreshWorker();
+			}
+
+			if ( worker.lastTouch > Date.now() - RUN_WORKER_TIMEOUT ) {
+				return tick();
+			}
+
+			refreshes++;
+
+			if ( refreshes >= MAX_WORKER_REFRESHES ) {
+				if ( restarts < MAX_WORKER_RESTARTS ) {
+					if ( verbose ) {
+						console.log(
+							`Worker ${ worker.id } not acknowledged after ${
+								ACKNOWLEDGE_WORKER_TIMEOUT / 1000
+							}s.`
+						);
+					}
+					return retryWorker().then( resolve, reject );
+				}
+				await cleanupWorker( reportId, verbose );
+				return reject(
+					new Error(
+						`Worker ${ worker.id } for test ${ reportId } timed out after ${ MAX_WORKER_RESTARTS } restarts.`
+					)
+				);
+			}
+
+			if ( verbose ) {
+				console.log(
+					`\nRefreshing worker ${ worker.id } for test ${ reportId }...${ refreshes }`
+				);
+			}
+
+			return refreshWorker();
+		}
+
+		function tick() {
+			setTimeout( checkWorker, POLL_WORKER_TIMEOUT );
+		}
+
+		checkWorker();
+	} );
+}