Tests: migrate testing infrastructure to minimal dependencies

This is a complete rework of our testing infrastructure. The main goal is to modernize and drop deprecated or undermaintained dependencies (specifically, grunt, karma, and testswarm). We've achieved that by limiting our dependency list to ones that are unlikely to drop support any time soon. The new dependency list includes:

- `qunit` (our trusty unit testing library)
- `selenium-webdriver` (for spinning up local browsers)
- `express` (for starting a test server and adding middleware)
  - express middleware includes uses of `body-parser` and `raw-body`
- `yargs` (for constructing a CLI with pretty help text)
- BrowserStack (for running each of our QUnit modules separately in all of our supported browsers)
  - `browserstack-local` (for opening a local tunnel. This is the same package still currently used in the new Browserstack SDK)
  - We are not using any other BrowserStack library. The newest BrowserStack SDK does not fit our needs (and isn't open source). Existing libraries, such as `node-browserstack` or `browserstack-runner`, either do not quite fit our needs, are under-maintained and out-of-date, or are not robust enough to meet all of our requirements. We instead call the [BrowserStack REST API](https://github.com/browserstack/api) directly.

## BrowserStack Runner
- automatically retries individual modules in case of test failure(s)
- automatically attempts to re-establish broken tunnels
- automatically refreshes the page in case a test run has stalled
- runs all browsers concurrently and uses as many sessions as are available under the BrowserStack plan. It will wait for available sessions if there are none.
- supports filtering the available list of browsers by browser name, browser version, device, OS, and OS version (see `npm run test:unit -- --list-browsers` for more info). It will retrieve the latest matching browser available if any of those parameters are not specified.
- cleans up after itself (closes the local tunnel, stops the test server, etc.)
- Requires `BROWSERSTACK_USERNAME` and `BROWSERSTACK_ACCESS_KEY` environment variables.

## Selenium Runner
- supports running any local browser as long as the driver is installed, including support for headless mode in Chrome, FF, and Edge
- supports running `basic` tests on the latest [jsdom](https://github.com/jsdom/jsdom#readme), which can be seen in action in this PR (see `test:browserless`)
- Node tests will run as before in PRs and all non-dependabot branches, but now includes tests on real Safari in a GH actions macos image instead of playwright-webkit.
- can run multiple browsers and multiple modules concurrently

Other notes:
- Stale dependencies have been removed and all remaining dependencies have been upgraded with a few exceptions:
  - `sinon`: stopped supporting IE in version 10. But, `sinon` has been updated to 9.x.
  - `husky`: latest does not support Node 10 and runs on `npm install`. Needed for now until git builds are migrated to GitHub Actions.
  - `rollup`: latest does not support Node 10. Needed for now until git builds are migrated to GitHub Actions.
- BrowserStack tests are set to run on each `main` branch commit
- `debug` mode leaves Selenium browsers open whether they pass or fail and leaves browsers with test failures open on BrowserStack. The latter is to avoid leaving open too many sessions.
- This PR includes a workflow to dispatch BrowserStack runs on-demand
- The Node version used for most workflow tests has been upgraded to 20.x
- updated supportjQuery to 3.7.1

Run `npm run test:unit -- --help` for CLI documentation

Close gh-5418

5 files changed, 616 insertions, 0 deletions

diff --git a/test/runner/browserstack/api.js b/test/runner/browserstack/api.js
new file mode 100644
index 000000000..40982c9b3
--- /dev/null
+++ b/test/runner/browserstack/api.js
@@ -0,0 +1,289 @@
+/**
+ * Browserstack API is documented at
+ * https://github.com/browserstack/api
+ */
+
+import { createAuthHeader } from "./createAuthHeader.js";
+
+const browserstackApi = "https://api.browserstack.com";
+const apiVersion = 5;
+
+const username = process.env.BROWSERSTACK_USERNAME;
+const accessKey = process.env.BROWSERSTACK_ACCESS_KEY;
+
+// iOS has null for version numbers,
+// and we do not need a similar check for OS versions.
+const rfinalVersion = /(?:^[0-9\.]+$)|(?:^null$)/;
+
+const rnonDigits = /(?:[^\d\.]+)|(?:20\d{2})/g;
+
+async function fetchAPI( path, options = {}, versioned = true ) {
+	if ( !username || !accessKey ) {
+		throw new Error(
+			"BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables must be set."
+		);
+	}
+	const init = {
+		method: "GET",
+		...options,
+		headers: {
+			authorization: createAuthHeader( username, accessKey ),
+			accept: "application/json",
+			"content-type": "application/json",
+			...options.headers
+		}
+	};
+	const response = await fetch(
+		`${ browserstackApi }/${ versioned ? `${ apiVersion }/` : "" }${ path }`,
+		init
+	);
+	if ( !response.ok ) {
+		console.log(
+			`\n${ init.method } ${ path }`,
+			response.status,
+			response.statusText
+		);
+		throw new Error( `Error fetching ${ path }` );
+	}
+	return response.json();
+}
+
+/**
+ * =============================
+ * Browsers API
+ * =============================
+ */
+
+function compareVersionNumbers( a, b ) {
+	if ( a != null && b == null ) {
+		return -1;
+	}
+	if ( a == null && b != null ) {
+		return 1;
+	}
+	if ( a == null && b == null ) {
+		return 0;
+	}
+	const aParts = a.replace( rnonDigits, "" ).split( "." );
+	const bParts = b.replace( rnonDigits, "" ).split( "." );
+
+	if ( aParts.length > bParts.length ) {
+		return -1;
+	}
+	if ( aParts.length < bParts.length ) {
+		return 1;
+	}
+
+	for ( let i = 0; i < aParts.length; i++ ) {
+		const aPart = Number( aParts[ i ] );
+		const bPart = Number( bParts[ i ] );
+		if ( aPart < bPart ) {
+			return -1;
+		}
+		if ( aPart > bPart ) {
+			return 1;
+		}
+	}
+}
+
+function sortBrowsers( a, b ) {
+	if ( a.browser < b.browser ) {
+		return -1;
+	}
+	if ( a.browser > b.browser ) {
+		return 1;
+	}
+	const browserComparison = compareVersionNumbers(
+		a.browser_version,
+		b.browser_version
+	);
+	if ( browserComparison ) {
+		return browserComparison;
+	}
+	if ( a.os < b.os ) {
+		return -1;
+	}
+	if ( a.os > b.os ) {
+		return 1;
+	}
+	const osComparison = compareVersionNumbers( a.os_version, b.os_version );
+	if ( osComparison ) {
+		return osComparison;
+	}
+	const deviceComparison = compareVersionNumbers( a.device, b.device );
+	if ( deviceComparison ) {
+		return deviceComparison;
+	}
+	return 0;
+}
+
+export async function getBrowsers( { flat = false } = {} ) {
+	const query = new URLSearchParams();
+	if ( flat ) {
+		query.append( "flat", true );
+	}
+	const browsers = await fetchAPI( `/browsers?${ query }` );
+	return browsers.sort( sortBrowsers );
+}
+
+function matchVersion( browserVersion, version ) {
+	if ( !version ) {
+		return false;
+	}
+	const regex = new RegExp(
+		`^${ version.replace( /\\/g, "\\\\" ).replace( /\./g, "\\." ) }\\b`,
+		"i"
+	);
+	return regex.test( browserVersion );
+}
+
+export async function filterBrowsers( filter ) {
+	const browsers = await getBrowsers( { flat: true } );
+	if ( !filter ) {
+		return browsers;
+	}
+	const filterBrowser = ( filter.browser ?? "" ).toLowerCase();
+	const filterVersion = ( filter.browser_version ?? "" ).toLowerCase();
+	const filterOs = ( filter.os ?? "" ).toLowerCase();
+	const filterOsVersion = ( filter.os_version ?? "" ).toLowerCase();
+	const filterDevice = ( filter.device ?? "" ).toLowerCase();
+
+	return browsers.filter( ( browser ) => {
+		return (
+			( !filterBrowser || filterBrowser === browser.browser.toLowerCase() ) &&
+			( !filterVersion ||
+				matchVersion( browser.browser_version, filterVersion ) ) &&
+			( !filterOs || filterOs === browser.os.toLowerCase() ) &&
+			( !filterOsVersion ||
+				filterOsVersion === browser.os_version.toLowerCase() ) &&
+			( !filterDevice || filterDevice === ( browser.device || "" ).toLowerCase() )
+		);
+	} );
+}
+
+export async function listBrowsers( filter ) {
+	const browsers = await filterBrowsers( filter );
+	console.log( "Available browsers:" );
+	for ( const browser of browsers ) {
+		let message = `    ${ browser.browser }_`;
+		if ( browser.device ) {
+			message += `:${ browser.device }_`;
+		} else {
+			message += `${ browser.browser_version }_`;
+		}
+		message += `${ browser.os }_${ browser.os_version }`;
+		console.log( message );
+	}
+}
+
+export async function getLatestBrowser( filter ) {
+	const browsers = await filterBrowsers( filter );
+
+	// The list is sorted in ascending order,
+	// so the last item is the latest.
+	return browsers.findLast( ( browser ) =>
+		rfinalVersion.test( browser.browser_version )
+	);
+}
+
+/**
+ * =============================
+ * Workers API
+ * =============================
+ */
+
+/**
+ * A browser object may only have one of `browser` or `device` set;
+ * which property is set will depend on `os`.
+ *
+ * `options`: is an object with the following properties:
+ *   `os`: The operating system.
+ *   `os_version`: The operating system version.
+ *   `browser`: The browser name.
+ *   `browser_version`: The browser version.
+ *   `device`: The device name.
+ *   `url` (optional): Which URL to navigate to upon creation.
+ *   `timeout` (optional): Maximum life of the worker (in seconds). Maximum value of `1800`. Specifying `0` will use the default of `300`.
+ *   `name` (optional): Provide a name for the worker.
+ *   `build` (optional): Group workers into a build.
+ *   `project` (optional): Provide the project the worker belongs to.
+ *   `resolution` (optional): Specify the screen resolution (e.g. "1024x768").
+ *   `browserstack.local` (optional): Set to `true` to mark as local testing.
+ *   `browserstack.video` (optional): Set to `false` to disable video recording.
+ *   `browserstack.localIdentifier` (optional): ID of the local tunnel.
+ */
+export function createWorker( options ) {
+	return fetchAPI( "/worker", {
+		method: "POST",
+		body: JSON.stringify( options )
+	} );
+}
+
+/**
+ * Returns a worker object, if one exists, with the following properties:
+ *   `id`: The worker id.
+ *   `status`: A string representing the current status of the worker.
+ *     Possible statuses: `"running"`, `"queue"`.
+ */
+export function getWorker( id ) {
+	return fetchAPI( `/worker/${ id }` );
+}
+
+export async function deleteWorker( id, verbose ) {
+	await fetchAPI( `/worker/${ id }`, { method: "DELETE" } );
+	if ( verbose ) {
+		console.log( `\nWorker ${ id } stopped.` );
+	}
+}
+
+export function getWorkers() {
+	return fetchAPI( "/workers" );
+}
+
+/**
+ * Change the URL of a worker,
+ * or refresh if it's the same URL.
+ */
+export function changeUrl( id, url ) {
+	return fetchAPI( `/worker/${ id }/url.json`, {
+		method: "PUT",
+		body: JSON.stringify( {
+			timeout: 20,
+			url: encodeURI( url )
+		} )
+	} );
+}
+
+/**
+ * Stop all workers
+ */
+export async function stopWorkers() {
+	const workers = await getWorkers();
+
+	// Run each request on its own
+	// to avoid connect timeout errors.
+	for ( const worker of workers ) {
+		try {
+			await deleteWorker( worker.id, true );
+		} catch ( error ) {
+
+			// Log the error, but continue trying to remove workers.
+			console.error( error );
+		}
+	}
+}
+
+/**
+ * =============================
+ * Plan API
+ * =============================
+ */
+
+export function getPlan() {
+	return fetchAPI( "/automate/plan.json", {}, false );
+}
+
+export async function getAvailableSessions() {
+	const [ plan, workers ] = await Promise.all( [ getPlan(), getWorkers() ] );
+	return plan.parallel_sessions_max_allowed - workers.length;
+}
diff --git a/test/runner/browserstack/buildBrowserFromString.js b/test/runner/browserstack/buildBrowserFromString.js
new file mode 100644
index 000000000..55aa38053
--- /dev/null
+++ b/test/runner/browserstack/buildBrowserFromString.js
@@ -0,0 +1,10 @@
+export function buildBrowserFromString( str ) {
+	const [ browser, versionOrDevice, os, osVersion ] = str.split( "_" );
+
+	// If the version starts with a colon, it's a device
+	if ( versionOrDevice && versionOrDevice.startsWith( ":" ) ) {
+		return { browser, device: versionOrDevice.slice( 1 ), os, os_version: osVersion };
+	}
+
+	return { browser, browser_version: versionOrDevice, os, os_version: osVersion };
+}
diff --git a/test/runner/browserstack/createAuthHeader.js b/test/runner/browserstack/createAuthHeader.js
new file mode 100644
index 000000000..fe4831e9a
--- /dev/null
+++ b/test/runner/browserstack/createAuthHeader.js
@@ -0,0 +1,7 @@
+const textEncoder = new TextEncoder();
+
+export function createAuthHeader( username, accessKey ) {
+	const encoded = textEncoder.encode( `${ username }:${ accessKey }` );
+	const base64 = btoa( String.fromCodePoint.apply( null, encoded ) );
+	return `Basic ${ base64 }`;
+}
diff --git a/test/runner/browserstack/local.js b/test/runner/browserstack/local.js
new file mode 100644
index 000000000..c84cf155c
--- /dev/null
+++ b/test/runner/browserstack/local.js
@@ -0,0 +1,34 @@
+import browserstackLocal from "browserstack-local";
+
+export async function localTunnel( localIdentifier, opts = {} ) {
+	const tunnel = new browserstackLocal.Local();
+
+	return new Promise( ( resolve, reject ) => {
+
+		// https://www.browserstack.com/docs/local-testing/binary-params
+		tunnel.start(
+			{
+				"enable-logging-for-api": "",
+				localIdentifier,
+				...opts
+			},
+			async( error ) => {
+				if ( error || !tunnel.isRunning() ) {
+					return reject( error );
+				}
+				resolve( {
+					stop: function stopTunnel() {
+						return new Promise( ( resolve, reject ) => {
+							tunnel.stop( ( error ) => {
+								if ( error ) {
+									return reject( error );
+								}
+								resolve();
+							} );
+						} );
+					}
+				} );
+			}
+		);
+	} );
+}
diff --git a/test/runner/browserstack/workers.js b/test/runner/browserstack/workers.js
new file mode 100644
index 000000000..8f0ab68f0
--- /dev/null
+++ b/test/runner/browserstack/workers.js
@@ -0,0 +1,276 @@
+import chalk from "chalk";
+import { getBrowserString } from "../lib/getBrowserString.js";
+import { changeUrl, createWorker, deleteWorker, getWorker } from "./api.js";
+
+const workers = Object.create( null );
+
+// Acknowledge the worker within the time limit.
+// BrowserStack can take much longer spinning up
+// some browsers, such as iOS 15 Safari.
+const ACKNOWLEDGE_WORKER_TIMEOUT = 60 * 1000 * 8;
+const ACKNOWLEDGE_WORKER_INTERVAL = 1000;
+
+// No report after the time limit
+// should refresh the worker
+const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
+const MAX_WORKER_RESTARTS = 5;
+const MAX_WORKER_REFRESHES = 1;
+const POLL_WORKER_TIMEOUT = 1000;
+
+export async function cleanupWorker( reportId, verbose ) {
+	const worker = workers[ reportId ];
+	if ( worker ) {
+		try {
+			delete workers[ reportId ];
+			await deleteWorker( worker.id, verbose );
+		} catch ( error ) {
+			console.error( error );
+		}
+	}
+}
+
+export function debugWorker( reportId ) {
+	const worker = workers[ reportId ];
+	if ( worker ) {
+		worker.debug = true;
+	}
+}
+
+/**
+ * Set the last time a request was
+ * received related to the worker.
+ */
+export function touchWorker( reportId ) {
+	const worker = workers[ reportId ];
+	if ( worker ) {
+		worker.lastTouch = Date.now();
+	}
+}
+
+export function retryTest( reportId, retries ) {
+	const worker = workers[ reportId ];
+	if ( worker ) {
+		worker.retries ||= 0;
+		worker.retries++;
+		if ( worker.retries <= retries ) {
+			worker.retry = true;
+			console.log( `\nRetrying test ${ reportId }...${ worker.retries }` );
+			return true;
+		}
+	}
+	return false;
+}
+
+export async function cleanupAllWorkers( verbose ) {
+	const workersRemaining = Object.keys( workers ).length;
+	if ( workersRemaining ) {
+		if ( verbose ) {
+			console.log(
+				`Stopping ${ workersRemaining } stray worker${
+					workersRemaining > 1 ? "s" : ""
+				}...`
+			);
+		}
+		await Promise.all(
+			Object.values( workers ).map( ( worker ) => deleteWorker( worker.id, verbose ) )
+		);
+	}
+}
+
+async function waitForAck( id, verbose ) {
+	return new Promise( ( resolve, reject ) => {
+		const interval = setInterval( () => {
+			const worker = workers[ id ];
+			if ( !worker ) {
+				clearTimeout( timeout );
+				clearInterval( interval );
+				return reject( new Error( `Worker ${ id } not found.` ) );
+			}
+			if ( worker.lastTouch ) {
+				if ( verbose ) {
+					console.log( `\nWorker ${ id } acknowledged.` );
+				}
+				clearTimeout( timeout );
+				clearInterval( interval );
+				resolve();
+			}
+		}, ACKNOWLEDGE_WORKER_INTERVAL );
+		const timeout = setTimeout( () => {
+			clearInterval( interval );
+			const worker = workers[ id ];
+			reject(
+				new Error(
+					`Worker ${
+						worker ? worker.id : ""
+					} for test ${ id } not acknowledged after ${
+						ACKNOWLEDGE_WORKER_TIMEOUT / 1000
+					}s.`
+				)
+			);
+		}, ACKNOWLEDGE_WORKER_TIMEOUT );
+	} );
+}
+
+export async function runWorker(
+	url,
+	browser,
+	options,
+	restarts = 0
+) {
+	const { modules, reportId, runId, verbose } = options;
+	const worker = await createWorker( {
+		...browser,
+		url: encodeURI( url ),
+		project: "jquery",
+		build: `Run ${ runId }`,
+		name: `${ modules.join( "," ) } (${ reportId })`,
+
+		// Set the max here, so that we can
+		// control the timeout
+		timeout: 1800,
+
+		// Not documented in the API docs,
+		// but required to make local testing work.
+		// See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
+		"browserstack.local": true,
+		"browserstack.localIdentifier": runId
+	} );
+
+	workers[ reportId ] = worker;
+
+	const timeMessage = `\nWorker ${
+		worker.id
+	} created for test ${ reportId } (${ chalk.yellow( getBrowserString( browser ) ) })`;
+
+	if ( verbose ) {
+		console.time( timeMessage );
+	}
+
+	async function retryWorker() {
+		await cleanupWorker( reportId, verbose );
+		if ( verbose ) {
+			console.log( `Retrying worker for test ${ reportId }...${ restarts + 1 }` );
+		}
+		return runWorker( url, browser, options, restarts + 1 );
+	}
+
+	// Wait for the worker to be acknowledged
+	try {
+		await waitForAck( reportId );
+	} catch ( error ) {
+		if ( !workers[ reportId ] ) {
+
+			// The worker has already been cleaned up
+			return;
+		}
+
+		if ( restarts < MAX_WORKER_RESTARTS ) {
+			return retryWorker();
+		}
+
+		throw error;
+	}
+
+	if ( verbose ) {
+		console.timeEnd( timeMessage );
+	}
+
+	let refreshes = 0;
+	let loggedStarted = false;
+	return new Promise( ( resolve, reject ) => {
+		async function refreshWorker() {
+			try {
+				await changeUrl( worker.id, url );
+				touchWorker( reportId );
+				return tick();
+			} catch ( error ) {
+				if ( !workers[ reportId ] ) {
+
+					// The worker has already been cleaned up
+					return resolve();
+				}
+				console.error( error );
+				return retryWorker().then( resolve, reject );
+			}
+		}
+
+		async function checkWorker() {
+			const worker = workers[ reportId ];
+
+			if ( !worker || worker.debug ) {
+				return resolve();
+			}
+
+			let fetchedWorker;
+			try {
+				fetchedWorker = await getWorker( worker.id );
+			} catch ( error ) {
+				return reject( error );
+			}
+			if (
+				!fetchedWorker ||
+				( fetchedWorker.status !== "running" && fetchedWorker.status !== "queue" )
+			) {
+				return resolve();
+			}
+
+			if ( verbose && !loggedStarted ) {
+				loggedStarted = true;
+				console.log(
+					`\nTest ${ chalk.bold( reportId ) } is ${
+						worker.status === "running" ? "running" : "in the queue"
+					}.`
+				);
+				console.log( `  View at ${ fetchedWorker.browser_url }.` );
+			}
+
+			// Refresh the worker if a retry is triggered
+			if ( worker.retry ) {
+				worker.retry = false;
+
+				// Reset recovery refreshes
+				refreshes = 0;
+				return refreshWorker();
+			}
+
+			if ( worker.lastTouch > Date.now() - RUN_WORKER_TIMEOUT ) {
+				return tick();
+			}
+
+			refreshes++;
+
+			if ( refreshes >= MAX_WORKER_REFRESHES ) {
+				if ( restarts < MAX_WORKER_RESTARTS ) {
+					if ( verbose ) {
+						console.log(
+							`Worker ${ worker.id } not acknowledged after ${
+								ACKNOWLEDGE_WORKER_TIMEOUT / 1000
+							}s.`
+						);
+					}
+					return retryWorker().then( resolve, reject );
+				}
+				await cleanupWorker( reportId, verbose );
+				return reject(
+					new Error(
+						`Worker ${ worker.id } for test ${ reportId } timed out after ${ MAX_WORKER_RESTARTS } restarts.`
+					)
+				);
+			}
+
+			if ( verbose ) {
+				console.log(
+					`\nRefreshing worker ${ worker.id } for test ${ reportId }...${ refreshes }`
+				);
+			}
+
+			return refreshWorker();
+		}
+
+		function tick() {
+			setTimeout( checkWorker, POLL_WORKER_TIMEOUT );
+		}
+
+		checkWorker();
+	} );
+}