Tests: replace grunt-contrib-qunit with jQuery test runner

- add filestash workflow

Close gh-2221

22 files changed, 1217 insertions, 13 deletions

diff --git a/tests/index.html b/tests/index.html
index d1465c192..268cdf190 100644
--- a/tests/index.html
+++ b/tests/index.html
@@ -18,8 +18,7 @@
 		<p><a href="unit/index.html">Unit tests</a> exist for all functionality in jQuery UI.
 		The unit tests can be run locally (some tests require a web server with PHP)
 		to ensure proper functionality before committing changes.
-		The unit tests are also run on <a href="https://swarm.jquery.org/project/jqueryui">TestSwarm</a>
-		for every commit.</p>
+		The unit tests are also run in Chrome, Firefox, Edge, and Safari on every commit.</p>
 
 		<h2>Visual Tests</h2>
 		<p><a href="visual/index.html">Visual tests</a> only exist in cases where we can't verify proper functionality
diff --git a/tests/lib/bootstrap.js b/tests/lib/bootstrap.js
index 98c17f4d5..fd9b1eb65 100644
--- a/tests/lib/bootstrap.js
+++ b/tests/lib/bootstrap.js
@@ -1,7 +1,7 @@
 ( function() {
 "use strict";
 
-var DEFAULT_JQUERY_VERSION = "3.7.0";
+var DEFAULT_JQUERY_VERSION = "3.7.1";
 
 requirejs.config( {
 	paths: {
@@ -11,7 +11,6 @@ requirejs.config( {
 		"jquery-migrate": migrateUrl(),
 		"jquery-simulate": "../../../external/jquery-simulate/jquery.simulate",
 		"lib": "../../lib",
-		"phantom-bridge": "../../../node_modules/grunt-contrib-qunit/phantomjs/bridge",
 		"qunit-assert-classes": "../../lib/vendor/qunit-assert-classes/qunit-assert-classes",
 		"qunit-assert-close": "../../lib/vendor/qunit-assert-close/qunit-assert-close",
 		"qunit": "../../../external/qunit/qunit",
@@ -33,11 +32,6 @@ define( "jquery-no-back-compat", [ "jquery" ], function( $ ) {
 	return $;
 } );
 
-// Create a dummy bridge if we're not actually testing in PhantomJS
-if ( !/PhantomJS/.test( navigator.userAgent ) ) {
-	define( "phantom-bridge", function() {} );
-}
-
 // Load all modules in series
 function requireModules( dependencies, callback, modules ) {
 	if ( !dependencies.length ) {
diff --git a/tests/lib/qunit.js b/tests/lib/qunit.js
index 7049d60a5..eac9c4a92 100644
--- a/tests/lib/qunit.js
+++ b/tests/lib/qunit.js
@@ -3,8 +3,7 @@ define( [
 	"jquery",
 	"qunit-assert-classes",
 	"qunit-assert-close",
-	"lib/qunit-assert-domequal",
-	"phantom-bridge"
+	"lib/qunit-assert-domequal"
 ], function( QUnit, $ ) {
 "use strict";
 
@@ -14,6 +13,8 @@ QUnit.config.requireExpects = true;
 QUnit.config.urlConfig.push( {
 	id: "jquery",
 	label: "jQuery version",
+
+	// Keep in sync with tests/runner/jquery.js
 	value: [
 		"1.8.0", "1.8.1", "1.8.2", "1.8.3",
 		"1.9.0", "1.9.1",
@@ -30,7 +31,7 @@ QUnit.config.urlConfig.push( {
 		"3.4.0", "3.4.1",
 		"3.5.0", "3.5.1",
 		"3.6.0", "3.6.1", "3.6.2", "3.6.3", "3.6.4",
-		"3.7.0",
+		"3.7.0", "3.7.1",
 		"3.x-git", "git", "custom"
 	],
 	tooltip: "Which jQuery Core version to test against"
diff --git a/tests/runner/.eslintrc.json b/tests/runner/.eslintrc.json
new file mode 100644
index 000000000..9dc38dbd7
--- /dev/null
+++ b/tests/runner/.eslintrc.json
@@ -0,0 +1,41 @@
+{
+	"root": true,
+
+	"extends": "jquery",
+
+	"overrides": [
+		{
+			"files": ["**/*"],
+			"env": {
+				"node": true
+			},
+			"globals": {
+				"fetch": false,
+				"Promise": false,
+				"require": false
+			},
+			"parserOptions": {
+				"ecmaVersion": 2022,
+				"sourceType": "module"
+			}
+		},
+		{
+			"files": ["./listeners.js"],
+			"env": {
+				"browser": true,
+				"node": false
+			},
+			"globals": {
+				"QUnit": false,
+				"Symbol": false
+			},
+			"parserOptions": {
+				"ecmaVersion": 5,
+				"sourceType": "script"
+			},
+			"rules": {
+				"strict": ["error", "function"]
+			}
+		}
+	]
+}
diff --git a/tests/runner/browsers.js b/tests/runner/browsers.js
new file mode 100644
index 000000000..4160ac0b5
--- /dev/null
+++ b/tests/runner/browsers.js
@@ -0,0 +1,4 @@
+// This list is static, so no requests are required
+// in the command help menu.
+
+export const browsers = [ "chrome", "ie", "firefox", "edge", "safari", "opera" ];
diff --git a/tests/runner/command.js b/tests/runner/command.js
new file mode 100644
index 000000000..655024fb4
--- /dev/null
+++ b/tests/runner/command.js
@@ -0,0 +1,78 @@
+import yargs from "yargs/yargs";
+import { browsers } from "./browsers.js";
+import { suites } from "./suites.js";
+import { run } from "./run.js";
+import { jquery } from "./jquery.js";
+
+const argv = yargs( process.argv.slice( 2 ) )
+	.version( false )
+	.strict()
+	.command( {
+		command: "[options]",
+		describe: "Run jQuery tests in a browser"
+	} )
+	.option( "suite", {
+		alias: "s",
+		type: "array",
+		choices: suites,
+		description:
+			"Run tests for a specific test suite.\n" +
+			"Pass multiple test suites by repeating the option.\n" +
+			"Defaults to all suites."
+	} )
+	.option( "jquery", {
+		alias: "j",
+		type: "array",
+		choices: jquery,
+		description:
+			"Run tests against a specific jQuery version.\n" +
+			"Pass multiple versions by repeating the option.",
+		default: [ "3.7.1" ]
+	} )
+	.option( "migrate", {
+		type: "boolean",
+		description:
+			"Run tests with jQuery Migrate enabled.",
+		default: false
+	} )
+	.option( "browser", {
+		alias: "b",
+		type: "array",
+		choices: browsers,
+		description:
+			"Run tests in a specific browser.\n" +
+			"Pass multiple browsers by repeating the option.",
+		default: [ "chrome" ]
+	} )
+	.option( "headless", {
+		alias: "h",
+		type: "boolean",
+		description:
+			"Run tests in headless mode. Cannot be used with --debug.",
+		conflicts: [ "debug" ]
+	} )
+	.option( "debug", {
+		alias: "d",
+		type: "boolean",
+		description:
+			"Leave the browser open for debugging. Cannot be used with --headless.",
+		conflicts: [ "headless" ]
+	} )
+	.option( "retries", {
+		alias: "r",
+		type: "number",
+		description: "Number of times to retry failed tests."
+	} )
+	.option( "concurrency", {
+		alias: "c",
+		type: "number",
+		description: "Run tests in parallel in multiple browsers. Defaults to 8."
+	} )
+	.option( "verbose", {
+		alias: "v",
+		type: "boolean",
+		description: "Log additional information."
+	} )
+	.help().argv;
+
+run( argv );
diff --git a/tests/runner/createTestServer.js b/tests/runner/createTestServer.js
new file mode 100644
index 000000000..878aa7d83
--- /dev/null
+++ b/tests/runner/createTestServer.js
@@ -0,0 +1,66 @@
+import bodyParser from "body-parser";
+import express from "express";
+import bodyParserErrorHandler from "express-body-parser-error-handler";
+import { readFile } from "node:fs/promises";
+
+export async function createTestServer( report ) {
+	const app = express();
+
+	// Redirect home to test page
+	app.get( "/", ( _req, res ) => {
+		res.redirect( "/tests/" );
+	} );
+
+	// Redirect to trailing slash
+	app.use( ( req, res, next ) => {
+		if ( req.path === "/tests" ) {
+			const query = req.url.slice( req.path.length );
+			res.redirect( 301, `${ req.path }/${ query }` );
+		} else {
+			next();
+		}
+	} );
+
+	// Add a script tag to HTML pages to load the QUnit listeners
+	app.use( /\/tests\/unit\/([^/]+)\/\1\.html$/, async( req, res ) => {
+		const html = await readFile(
+			`tests/unit/${ req.params[ 0 ] }/${ req.params[ 0 ] }.html`,
+			"utf8"
+		);
+		res.send(
+			html.replace(
+				"</head>",
+				"<script src=\"/tests/runner/listeners.js\"></script></head>"
+			)
+		);
+	} );
+
+	// Bind the reporter
+	app.post(
+		"/api/report",
+		bodyParser.json( { limit: "50mb" } ),
+		async( req, res ) => {
+			if ( report ) {
+				const response = await report( req.body );
+				if ( response ) {
+					res.json( response );
+					return;
+				}
+			}
+			res.sendStatus( 204 );
+		}
+	);
+
+	// Handle errors from the body parser
+	app.use( bodyParserErrorHandler() );
+
+	// Serve static files
+	app.use( "/dist", express.static( "dist" ) );
+	app.use( "/src", express.static( "src" ) );
+	app.use( "/tests", express.static( "tests" ) );
+	app.use( "/ui", express.static( "ui" ) );
+	app.use( "/themes", express.static( "themes" ) );
+	app.use( "/external", express.static( "external" ) );
+
+	return app;
+}
diff --git a/tests/runner/jquery.js b/tests/runner/jquery.js
new file mode 100644
index 000000000..3dee6269e
--- /dev/null
+++ b/tests/runner/jquery.js
@@ -0,0 +1,20 @@
+// Keep in sync with tests/lib/qunit.js
+export const jquery = [
+	"1.8.0", "1.8.1", "1.8.2", "1.8.3",
+	"1.9.0", "1.9.1",
+	"1.10.0", "1.10.1", "1.10.2",
+	"1.11.0", "1.11.1", "1.11.2", "1.11.3",
+	"1.12.0", "1.12.1", "1.12.2", "1.12.3", "1.12.4",
+	"2.0.0", "2.0.1", "2.0.2", "2.0.3",
+	"2.1.0", "2.1.1", "2.1.2", "2.1.3", "2.1.4",
+	"2.2.0", "2.2.1", "2.2.2", "2.2.3", "2.2.4",
+	"3.0.0",
+	"3.1.0", "3.1.1",
+	"3.2.0", "3.2.1",
+	"3.3.0", "3.3.1",
+	"3.4.0", "3.4.1",
+	"3.5.0", "3.5.1",
+	"3.6.0", "3.6.1", "3.6.2", "3.6.3", "3.6.4",
+	"3.7.0", "3.7.1",
+	"3.x-git", "git", "custom"
+];
diff --git a/tests/runner/lib/buildTestUrl.js b/tests/runner/lib/buildTestUrl.js
new file mode 100644
index 000000000..826548852
--- /dev/null
+++ b/tests/runner/lib/buildTestUrl.js
@@ -0,0 +1,21 @@
+export function buildTestUrl( suite, { jquery, migrate, port, reportId } ) {
+	if ( !port ) {
+		throw new Error( "No port specified." );
+	}
+
+	const query = new URLSearchParams();
+
+	if ( jquery ) {
+		query.append( "jquery", jquery );
+	}
+
+	if ( migrate ) {
+		query.append( "migrate", "true" );
+	}
+
+	if ( reportId ) {
+		query.append( "reportId", reportId );
+	}
+
+	return `http://localhost:${ port }/tests/unit/${ suite }/${ suite }.html?${ query }`;
+}
diff --git a/tests/runner/lib/generateHash.js b/tests/runner/lib/generateHash.js
new file mode 100644
index 000000000..66f2161d5
--- /dev/null
+++ b/tests/runner/lib/generateHash.js
@@ -0,0 +1,10 @@
+import crypto from "node:crypto";
+
+export function generateHash( string ) {
+	const hash = crypto.createHash( "md5" );
+	hash.update( string );
+
+	// QUnit hashes are 8 characters long
+	// We use 10 characters to be more visually distinct
+	return hash.digest( "hex" ).slice( 0, 10 );
+}
diff --git a/tests/runner/lib/getBrowserString.js b/tests/runner/lib/getBrowserString.js
new file mode 100644
index 000000000..413a60500
--- /dev/null
+++ b/tests/runner/lib/getBrowserString.js
@@ -0,0 +1,49 @@
+const browserMap = {
+	chrome: "Chrome",
+	edge: "Edge",
+	firefox: "Firefox",
+	ie: "IE",
+	jsdom: "JSDOM",
+	opera: "Opera",
+	safari: "Safari"
+};
+
+export function browserSupportsHeadless( browser ) {
+	browser = browser.toLowerCase();
+	return (
+		browser === "chrome" ||
+		browser === "firefox" ||
+		browser === "edge"
+	);
+}
+
+export function getBrowserString(
+	{
+		browser,
+		browser_version: browserVersion,
+		device,
+		os,
+		os_version: osVersion
+	},
+	headless
+) {
+	browser = browser.toLowerCase();
+	browser = browserMap[ browser ] || browser;
+	let str = browser;
+	if ( browserVersion ) {
+		str += ` ${ browserVersion }`;
+	}
+	if ( device ) {
+		str += ` for ${ device }`;
+	}
+	if ( os ) {
+		str += ` on ${ os }`;
+	}
+	if ( osVersion ) {
+		str += ` ${ osVersion }`;
+	}
+	if ( headless && browserSupportsHeadless( browser ) ) {
+		str += " (headless)";
+	}
+	return str;
+}
diff --git a/tests/runner/lib/prettyMs.js b/tests/runner/lib/prettyMs.js
new file mode 100644
index 000000000..99bae2b35
--- /dev/null
+++ b/tests/runner/lib/prettyMs.js
@@ -0,0 +1,18 @@
+/**
+ * Pretty print a time in milliseconds.
+ */
+export function prettyMs( time ) {
+	const minutes = Math.floor( time / 60000 );
+	const seconds = Math.floor( time / 1000 );
+	const ms = Math.floor( time % 1000 );
+
+	let prettyTime = `${ ms }ms`;
+	if ( seconds > 0 ) {
+		prettyTime = `${ seconds }s ${ prettyTime }`;
+	}
+	if ( minutes > 0 ) {
+		prettyTime = `${ minutes }m ${ prettyTime }`;
+	}
+
+	return prettyTime;
+}
diff --git a/tests/runner/listeners.js b/tests/runner/listeners.js
new file mode 100644
index 000000000..ed6fb24e8
--- /dev/null
+++ b/tests/runner/listeners.js
@@ -0,0 +1,112 @@
+( function() {
+	"use strict";
+
+	// Get the report ID from the URL.
+	var match = location.search.match( /reportId=([^&]+)/ );
+	if ( !match ) {
+		return;
+	}
+	var id = match[ 1 ];
+
+	// Adopted from https://github.com/douglascrockford/JSON-js
+	// Support: IE 11+
+	// Using the replacer argument of JSON.stringify in IE has issues
+	// TODO: Replace this with a circular replacer + JSON.stringify + WeakSet
+	function decycle( object ) {
+		var objects = [];
+
+		// The derez function recurses through the object, producing the deep copy.
+		function derez( value ) {
+			if (
+				typeof value === "object" &&
+				value !== null &&
+				!( value instanceof Boolean ) &&
+				!( value instanceof Date ) &&
+				!( value instanceof Number ) &&
+				!( value instanceof RegExp ) &&
+				!( value instanceof String )
+			) {
+
+				// Return a string early for elements
+				if ( value.nodeType ) {
+					return value.toString();
+				}
+
+				if ( objects.indexOf( value ) > -1 ) {
+					return;
+				}
+
+				objects.push( value );
+
+				if ( Array.isArray( value ) ) {
+
+					// If it is an array, replicate the array.
+					return value.map( derez );
+				} else {
+
+					// If it is an object, replicate the object.
+					var nu = Object.create( null );
+					Object.keys( value ).forEach( function( name ) {
+						nu[ name ] = derez( value[ name ] );
+					} );
+					return nu;
+				}
+			}
+
+			// Serialize Symbols as string representations so they are
+			// sent over the wire after being stringified.
+			if ( typeof value === "symbol" ) {
+
+				// We can *describe* unique symbols, but note that their identity
+				// (e.g., `Symbol() !== Symbol()`) is lost
+				var ctor = Symbol.keyFor( value ) !== undefined ? "Symbol.for" : "Symbol";
+				return ctor + "(" + JSON.stringify( value.description ) + ")";
+			}
+
+			return value;
+		}
+		return derez( object );
+	}
+
+	function send( type, data ) {
+		var json = JSON.stringify( {
+			id: id,
+			type: type,
+			data: data ? decycle( data ) : undefined
+		} );
+		var request = new XMLHttpRequest();
+		request.open( "POST", "/api/report", true );
+		request.setRequestHeader( "Content-Type", "application/json" );
+		request.send( json );
+		return request;
+	}
+
+	require( [ "qunit" ], function( QUnit ) {
+
+		// Send acknowledgement to the server.
+		send( "ack" );
+
+		QUnit.on( "testEnd", function( data ) {
+			send( "testEnd", data );
+		} );
+
+		QUnit.on( "runEnd", function( data ) {
+
+			// Reduce the payload size.
+			// childSuites is large and unused.
+			data.childSuites = undefined;
+
+			var request = send( "runEnd", data );
+			request.onload = function() {
+				if ( request.status === 200 && request.responseText ) {
+					try {
+						var json = JSON.parse( request.responseText );
+						window.location = json.url;
+					} catch ( e ) {
+						console.error( e );
+					}
+				}
+			};
+		} );
+	} );
+} )();
diff --git a/tests/runner/package.json b/tests/runner/package.json
new file mode 100644
index 000000000..bedb411a9
--- /dev/null
+++ b/tests/runner/package.json
@@ -0,0 +1,3 @@
+{
+	"type": "module"
+}
diff --git a/tests/runner/reporter.js b/tests/runner/reporter.js
new file mode 100644
index 000000000..392a2851b
--- /dev/null
+++ b/tests/runner/reporter.js
@@ -0,0 +1,134 @@
+import chalk from "chalk";
+import { getBrowserString } from "./lib/getBrowserString.js";
+import { prettyMs } from "./lib/prettyMs.js";
+import * as Diff from "diff";
+
+function serializeForDiff( value ) {
+
+	// Use naive serialization for everything except types with confusable values
+	if ( typeof value === "string" ) {
+		return JSON.stringify( value );
+	}
+	if ( typeof value === "bigint" ) {
+		return `${ value }n`;
+	}
+	return `${ value }`;
+}
+
+export function reportTest( test, reportId, { browser, headless } ) {
+	if ( test.status === "passed" ) {
+
+		// Write to console without newlines
+		process.stdout.write( "." );
+		return;
+	}
+
+	let message = `${ chalk.bold( `${ test.suiteName }: ${ test.name }` ) }`;
+	message += `\nTest ${ test.status } on ${ chalk.yellow(
+		getBrowserString( browser, headless )
+	) } (${ chalk.bold( reportId ) }).`;
+
+	// test.assertions only contains passed assertions;
+	// test.errors contains all failed asssertions
+	if ( test.errors.length ) {
+		for ( const error of test.errors ) {
+			message += "\n";
+			if ( error.message ) {
+				message += `\n${ error.message }`;
+			}
+			message += `\n${ chalk.gray( error.stack ) }`;
+
+			// Show expected and actual values
+			// if either is defined and non-null.
+			// error.actual is set to null for failed
+			// assert.expect() assertions, so skip those as well.
+			// This should be fine because error.expected would
+			// have to also be null for this to be skipped.
+			if ( error.expected != null || error.actual != null ) {
+				message += `\nexpected: ${ chalk.red( JSON.stringify( error.expected ) ) }`;
+				message += `\nactual: ${ chalk.green( JSON.stringify( error.actual ) ) }`;
+				let diff;
+
+				if ( Array.isArray( error.expected ) && Array.isArray( error.actual ) ) {
+
+					// Diff arrays
+					diff = Diff.diffArrays( error.expected, error.actual );
+				} else if (
+					typeof error.expected === "object" &&
+					typeof error.actual === "object"
+				) {
+
+					// Diff objects
+					diff = Diff.diffJson( error.expected, error.actual );
+				} else if (
+					typeof error.expected === "number" &&
+					typeof error.actual === "number"
+				) {
+
+					// Diff numbers directly
+					const value = error.actual - error.expected;
+					if ( value > 0 ) {
+						diff = [ { added: true, value: `+${ value }` } ];
+					} else {
+						diff = [ { removed: true, value: `${ value }` } ];
+					}
+				} else if (
+					typeof error.expected === "string" &&
+					typeof error.actual === "string"
+				) {
+
+					// Diff the characters of strings
+					diff = Diff.diffChars( error.expected, error.actual );
+				} else {
+
+					// Diff everything else as words
+					diff = Diff.diffWords(
+						serializeForDiff( error.expected ),
+						serializeForDiff( error.actual )
+					);
+				}
+
+				if ( diff ) {
+					message += "\n";
+					message += diff
+						.map( ( part ) => {
+							if ( part.added ) {
+								return chalk.green( part.value );
+							}
+							if ( part.removed ) {
+								return chalk.red( part.value );
+							}
+							return chalk.gray( part.value );
+						} )
+						.join( "" );
+				}
+			}
+		}
+	}
+
+	console.log( `\n\n${ message }` );
+
+	// Only return failed messages
+	if ( test.status === "failed" ) {
+		return message;
+	}
+}
+
+export function reportEnd( result, reportId, { browser, headless, jquery, migrate, suite } ) {
+	const fullBrowser = getBrowserString( browser, headless );
+	console.log(
+		`\n\nTests finished in ${ prettyMs( result.runtime ) } ` +
+			`for ${ chalk.yellow( suite ) } ` +
+			`and jQuery ${ chalk.yellow( jquery ) } ` +
+			( migrate ? `with ${ chalk.yellow( "jQuery Migrate enabled " ) }` : "" ) +
+			`in ${ chalk.yellow( fullBrowser ) } (${ chalk.bold( reportId ) })...`
+	);
+	console.log(
+		( result.status !== "passed" ?
+			`${ chalk.red( result.testCounts.failed ) } failed. ` :
+			"" ) +
+			`${ chalk.green( result.testCounts.total ) } passed. ` +
+			`${ chalk.gray( result.testCounts.skipped ) } skipped.`
+	);
+	return result.testCounts;
+}
diff --git a/tests/runner/run.js b/tests/runner/run.js
new file mode 100644
index 000000000..bf3a16191
--- /dev/null
+++ b/tests/runner/run.js
@@ -0,0 +1,234 @@
+import chalk from "chalk";
+import { asyncExitHook, gracefulExit } from "exit-hook";
+import { reportEnd, reportTest } from "./reporter.js";
+import { createTestServer } from "./createTestServer.js";
+import { buildTestUrl } from "./lib/buildTestUrl.js";
+import { generateHash } from "./lib/generateHash.js";
+import { getBrowserString } from "./lib/getBrowserString.js";
+import { suites as allSuites } from "./suites.js";
+import { cleanupAllBrowsers, touchBrowser } from "./selenium/browsers.js";
+import { addRun, getNextBrowserTest, retryTest, runAll } from "./selenium/queue.js";
+
+const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
+
+/**
+ * Run test suites in parallel in different browser instances.
+ */
+export async function run( {
+	browser: browserNames = [],
+	concurrency,
+	debug,
+	headless,
+	jquery: jquerys = [],
+	migrate,
+	retries = 0,
+	suite: suites = [],
+	verbose
+} ) {
+	if ( !browserNames.length ) {
+		browserNames = [ "chrome" ];
+	}
+	if ( !suites.length ) {
+		suites = allSuites;
+	}
+	if ( !jquerys.length ) {
+		jquerys = [ "3.7.1" ];
+	}
+	if ( headless && debug ) {
+		throw new Error(
+			"Cannot run in headless mode and debug mode at the same time."
+		);
+	}
+
+	const errorMessages = [];
+	const pendingErrors = {};
+
+	// Convert browser names to browser objects
+	let browsers = browserNames.map( ( b ) => ( { browser: b } ) );
+
+	// Create the test app and
+	// hook it up to the reporter
+	const reports = Object.create( null );
+	const app = await createTestServer( async( message ) => {
+		switch ( message.type ) {
+			case "testEnd": {
+				const reportId = message.id;
+				const report = reports[ reportId ];
+				touchBrowser( report.browser );
+				const errors = reportTest( message.data, reportId, report );
+				pendingErrors[ reportId ] ??= Object.create( null );
+				if ( errors ) {
+					pendingErrors[ reportId ][ message.data.name ] = errors;
+				} else {
+					const existing = pendingErrors[ reportId ][ message.data.name ];
+
+					// Show a message for flakey tests
+					if ( existing ) {
+						console.log();
+						console.warn(
+							chalk.italic(
+								chalk.gray( existing.replace( "Test failed", "Test flakey" ) )
+							)
+						);
+						console.log();
+						delete pendingErrors[ reportId ][ message.data.name ];
+					}
+				}
+				break;
+			}
+			case "runEnd": {
+				const reportId = message.id;
+				const report = reports[ reportId ];
+				touchBrowser( report.browser );
+				const { failed, total } = reportEnd(
+					message.data,
+					message.id,
+					reports[ reportId ]
+				);
+				report.total = total;
+
+				// Handle failure
+				if ( failed ) {
+					const retry = retryTest( reportId, retries );
+
+					// Retry if retryTest returns a test
+					if ( retry ) {
+						return retry;
+					}
+
+					errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
+				}
+
+				// Run the next test
+				return getNextBrowserTest( reportId );
+			}
+			case "ack": {
+				const report = reports[ message.id ];
+				touchBrowser( report.browser );
+				break;
+			}
+			default:
+				console.warn( "Received unknown message type:", message.type );
+		}
+	} );
+
+	// Start up local test server
+	let server;
+	let port;
+	await new Promise( ( resolve ) => {
+
+		// Pass 0 to choose a random, unused port
+		server = app.listen( 0, () => {
+			port = server.address().port;
+			resolve();
+		} );
+	} );
+
+	if ( !server || !port ) {
+		throw new Error( "Server not started." );
+	}
+
+	if ( verbose ) {
+		console.log( `Server started on port ${ port }.` );
+	}
+
+	function stopServer() {
+		return new Promise( ( resolve ) => {
+			server.close( () => {
+				if ( verbose ) {
+					console.log( "Server stopped." );
+				}
+				resolve();
+			} );
+		} );
+	}
+
+	asyncExitHook(
+		async() => {
+			await cleanupAllBrowsers( { verbose } );
+			await stopServer();
+		},
+		{ wait: EXIT_HOOK_WAIT_TIMEOUT }
+	);
+
+	function queueRuns( suite, browser ) {
+		const fullBrowser = getBrowserString( browser, headless );
+
+		for ( const jquery of jquerys ) {
+			const reportId = generateHash( `${ suite } ${ fullBrowser }` );
+			reports[ reportId ] = { browser, headless, jquery, migrate, suite };
+
+			const url = buildTestUrl( suite, {
+				jquery,
+				migrate,
+				port,
+				reportId
+			} );
+
+			const options = {
+				debug,
+				headless,
+				jquery,
+				migrate,
+				reportId,
+				suite,
+				verbose
+			};
+
+			addRun( url, browser, options );
+		}
+	}
+
+	for ( const browser of browsers ) {
+		for ( const suite of suites ) {
+			queueRuns( suite, browser );
+		}
+	}
+
+	try {
+		await runAll( { concurrency, verbose } );
+	} catch ( error ) {
+		console.error( error );
+		if ( !debug ) {
+			gracefulExit( 1 );
+		}
+	} finally {
+		console.log();
+		if ( errorMessages.length === 0 ) {
+			let stop = false;
+			for ( const report of Object.values( reports ) ) {
+				if ( !report.total ) {
+					stop = true;
+					console.error(
+						chalk.red(
+							`No tests were run for ${ report.suite } in ${ getBrowserString(
+								report.browser
+							) }`
+						)
+					);
+				}
+			}
+			if ( stop ) {
+				return gracefulExit( 1 );
+			}
+			console.log( chalk.green( "All tests passed!" ) );
+
+			if ( !debug ) {
+				gracefulExit( 0 );
+			}
+		} else {
+			console.error( chalk.red( `${ errorMessages.length } tests failed.` ) );
+			console.log(
+				errorMessages.map( ( error, i ) => `\n${ i + 1 }. ${ error }` ).join( "\n" )
+			);
+
+			if ( debug ) {
+				console.log();
+				console.log( "Leaving browsers open for debugging." );
+				console.log( "Press Ctrl+C to exit." );
+			} else {
+				gracefulExit( 1 );
+			}
+		}
+	}
+}
diff --git a/tests/runner/selenium/browsers.js b/tests/runner/selenium/browsers.js
new file mode 100644
index 000000000..568d6ed36
--- /dev/null
+++ b/tests/runner/selenium/browsers.js
@@ -0,0 +1,200 @@
+import chalk from "chalk";
+import { getBrowserString } from "../lib/getBrowserString.js";
+import createDriver from "./createDriver.js";
+
+const workers = Object.create( null );
+
+/**
+ * Keys are browser strings
+ * Structure of a worker:
+ * {
+ * 	 debug: boolean, // Stops the worker from being cleaned up when finished
+ *   id: string,
+ *   lastTouch: number, // The last time a request was received
+ *   url: string,
+ *   browser: object, // The browser object
+ *   options: object // The options to create the worker
+ * }
+ */
+
+// Acknowledge the worker within the time limit.
+const ACKNOWLEDGE_INTERVAL = 1000;
+const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 1;
+
+const MAX_WORKER_RESTARTS = 5;
+
+// No report after the time limit
+// should refresh the worker
+const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
+
+const WORKER_WAIT_TIME = 30000;
+
+// Limit concurrency to 8 by default in selenium
+const MAX_CONCURRENCY = 8;
+
+export function touchBrowser( browser ) {
+	const fullBrowser = getBrowserString( browser );
+	const worker = workers[ fullBrowser ];
+	if ( worker ) {
+		worker.lastTouch = Date.now();
+	}
+}
+
+async function waitForAck( worker, { fullBrowser, verbose } ) {
+	delete worker.lastTouch;
+	return new Promise( ( resolve, reject ) => {
+		const interval = setInterval( () => {
+			if ( worker.lastTouch ) {
+				if ( verbose ) {
+					console.log( `\n${ fullBrowser } acknowledged.` );
+				}
+				clearTimeout( timeout );
+				clearInterval( interval );
+				resolve();
+			}
+		}, ACKNOWLEDGE_INTERVAL );
+
+		const timeout = setTimeout( () => {
+			clearInterval( interval );
+			reject(
+				new Error(
+					`${ fullBrowser } not acknowledged after ${
+						ACKNOWLEDGE_TIMEOUT / 1000 / 60
+					}min.`
+				)
+			);
+		}, ACKNOWLEDGE_TIMEOUT );
+	} );
+}
+
+async function restartWorker( worker ) {
+	await cleanupWorker( worker, worker.options );
+	await createBrowserWorker(
+		worker.url,
+		worker.browser,
+		worker.options,
+		worker.restarts + 1
+	);
+}
+
+async function ensureAcknowledged( worker ) {
+	const fullBrowser = getBrowserString( worker.browser );
+	const verbose = worker.options.verbose;
+	try {
+		await waitForAck( worker, { fullBrowser, verbose } );
+		return worker;
+	} catch ( error ) {
+		console.error( error.message );
+		await restartWorker( worker );
+	}
+}
+
+export async function createBrowserWorker( url, browser, options, restarts = 0 ) {
+	if ( restarts > MAX_WORKER_RESTARTS ) {
+		throw new Error(
+			`Reached the maximum number of restarts for ${ chalk.yellow(
+				getBrowserString( browser )
+			) }`
+		);
+	}
+	const { concurrency = MAX_CONCURRENCY, debug, headless, verbose } = options;
+	while ( workers.length >= concurrency ) {
+		if ( verbose ) {
+			console.log( "\nWaiting for available sessions..." );
+		}
+		await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
+	}
+
+	const fullBrowser = getBrowserString( browser );
+
+	const driver = await createDriver( {
+		browserName: browser.browser,
+		headless,
+		url,
+		verbose
+	} );
+
+	const worker = {
+		debug: !!debug,
+		driver,
+		url,
+		browser,
+		restarts,
+		options
+	};
+
+	worker.debug = !!debug;
+	worker.url = url;
+	worker.browser = browser;
+	worker.restarts = restarts;
+	worker.options = options;
+	touchBrowser( browser );
+	workers[ fullBrowser ] = worker;
+
+	// Wait for the worker to show up in the list
+	// before returning it.
+	return ensureAcknowledged( worker );
+}
+
+export async function setBrowserWorkerUrl( browser, url ) {
+	const fullBrowser = getBrowserString( browser );
+	const worker = workers[ fullBrowser ];
+	if ( worker ) {
+		worker.url = url;
+	}
+}
+
+/**
+ * Checks that all browsers have received
+ * a response in the given amount of time.
+ * If not, the worker is restarted.
+ */
+export async function checkLastTouches() {
+	for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) {
+		if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) {
+			const options = worker.options;
+			if ( options.verbose ) {
+				console.log(
+					`\nNo response from ${ chalk.yellow( fullBrowser ) } in ${
+						RUN_WORKER_TIMEOUT / 1000 / 60
+					}min.`
+				);
+			}
+			await restartWorker( worker );
+		}
+	}
+}
+
+export async function cleanupWorker( worker, { verbose } ) {
+	for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
+		if ( w === worker ) {
+			delete workers[ fullBrowser ];
+			await w.driver.quit();
+			if ( verbose ) {
+				console.log( `\nStopped ${ fullBrowser }.` );
+			}
+			return;
+		}
+	}
+}
+
+export async function cleanupAllBrowsers( { verbose } ) {
+	const workersRemaining = Object.values( workers );
+	const numRemaining = workersRemaining.length;
+	if ( numRemaining ) {
+		try {
+			await Promise.all(
+				workersRemaining.map( ( worker ) => worker.driver.quit() )
+			);
+			if ( verbose ) {
+				console.log(
+					`Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
+				);
+			}
+		} catch ( error ) {
+
+			// Log the error, but do not consider the test run failed
+			console.error( error );
+		}
+	}
+}
diff --git a/tests/runner/selenium/createDriver.js b/tests/runner/selenium/createDriver.js
new file mode 100644
index 000000000..095c12214
--- /dev/null
+++ b/tests/runner/selenium/createDriver.js
@@ -0,0 +1,84 @@
+import { Builder, Capabilities, logging } from "selenium-webdriver";
+import Chrome from "selenium-webdriver/chrome.js";
+import Edge from "selenium-webdriver/edge.js";
+import Firefox from "selenium-webdriver/firefox.js";
+import { browserSupportsHeadless } from "../lib/getBrowserString.js";
+
+// Set script timeout to 10min
+const DRIVER_SCRIPT_TIMEOUT = 1000 * 60 * 10;
+
+export default async function createDriver( { browserName, headless, url, verbose } ) {
+	const capabilities = Capabilities[ browserName ]();
+	const prefs = new logging.Preferences();
+	prefs.setLevel( logging.Type.BROWSER, logging.Level.ALL );
+	capabilities.setLoggingPrefs( prefs );
+
+	let driver = new Builder().withCapabilities( capabilities );
+
+	const chromeOptions = new Chrome.Options();
+	chromeOptions.addArguments( "--enable-chrome-browser-cloud-management" );
+
+	// Alter the chrome binary path if
+	// the CHROME_BIN environment variable is set
+	if ( process.env.CHROME_BIN ) {
+		if ( verbose ) {
+			console.log( `Setting chrome binary to ${ process.env.CHROME_BIN }` );
+		}
+		chromeOptions.setChromeBinaryPath( process.env.CHROME_BIN );
+	}
+
+	const firefoxOptions = new Firefox.Options();
+
+	if ( process.env.FIREFOX_BIN ) {
+		if ( verbose ) {
+			console.log( `Setting firefox binary to ${ process.env.FIREFOX_BIN }` );
+		}
+
+		firefoxOptions.setBinary( process.env.FIREFOX_BIN );
+	}
+
+	const edgeOptions = new Edge.Options();
+	edgeOptions.addArguments( "--enable-chrome-browser-cloud-management" );
+
+	// Alter the edge binary path if
+	// the EDGE_BIN environment variable is set
+	if ( process.env.EDGE_BIN ) {
+		if ( verbose ) {
+			console.log( `Setting edge binary to ${ process.env.EDGE_BIN }` );
+		}
+		edgeOptions.setEdgeChromiumBinaryPath( process.env.EDGE_BIN );
+	}
+
+	if ( headless ) {
+		chromeOptions.addArguments( "--headless=new" );
+		firefoxOptions.addArguments( "--headless" );
+		edgeOptions.addArguments( "--headless=new" );
+		if ( !browserSupportsHeadless( browserName ) ) {
+			console.log(
+				`Headless mode is not supported for ${ browserName }.` +
+					"Running in normal mode instead."
+			);
+		}
+	}
+
+	driver = await driver
+		.setChromeOptions( chromeOptions )
+		.setFirefoxOptions( firefoxOptions )
+		.setEdgeOptions( edgeOptions )
+		.build();
+
+	if ( verbose ) {
+		const driverCapabilities = await driver.getCapabilities();
+		const name = driverCapabilities.getBrowserName();
+		const version = driverCapabilities.getBrowserVersion();
+		console.log( `\nDriver created for ${ name } ${ version }` );
+	}
+
+	// Increase script timeout to 10min
+	await driver.manage().setTimeouts( { script: DRIVER_SCRIPT_TIMEOUT } );
+
+	// Set the first URL for the browser
+	await driver.get( url );
+
+	return driver;
+}
diff --git a/tests/runner/selenium/queue.js b/tests/runner/selenium/queue.js
new file mode 100644
index 000000000..de24c5bb0
--- /dev/null
+++ b/tests/runner/selenium/queue.js
@@ -0,0 +1,97 @@
+import chalk from "chalk";
+import { getBrowserString } from "../lib/getBrowserString.js";
+import {
+	checkLastTouches,
+	createBrowserWorker,
+	setBrowserWorkerUrl
+} from "./browsers.js";
+
+const TEST_POLL_TIMEOUT = 1000;
+
+const queue = [];
+
+export function getNextBrowserTest( reportId ) {
+	const index = queue.findIndex( ( test ) => test.id === reportId );
+	if ( index === -1 ) {
+		return;
+	}
+
+	// Remove the completed test from the queue
+	const previousTest = queue[ index ];
+	queue.splice( index, 1 );
+
+	// Find the next test for the same browser
+	for ( const test of queue.slice( index ) ) {
+		if ( test.fullBrowser === previousTest.fullBrowser ) {
+
+			// Set the URL for our tracking
+			setBrowserWorkerUrl( test.browser, test.url );
+			test.running = true;
+
+			// Return the URL for the next test.
+			// listeners.js will use this to set the browser URL.
+			return { url: test.url };
+		}
+	}
+}
+
+export function retryTest( reportId, maxRetries ) {
+	if ( !maxRetries ) {
+		return;
+	}
+	const test = queue.find( ( test ) => test.id === reportId );
+	if ( test ) {
+		test.retries++;
+		if ( test.retries <= maxRetries ) {
+			console.log(
+				`\nRetrying test ${ reportId } for ${ chalk.yellow( test.options.suite ) }...${
+					test.retries
+				}`
+			);
+			return test;
+		}
+	}
+}
+
+export function addRun( url, browser, options ) {
+	queue.push( {
+		browser,
+		fullBrowser: getBrowserString( browser ),
+		id: options.reportId,
+		retries: 0,
+		url,
+		options,
+		running: false
+	} );
+}
+
+export async function runAll() {
+	return new Promise( async( resolve, reject ) => {
+		while ( queue.length ) {
+			try {
+				await checkLastTouches();
+			} catch ( error ) {
+				reject( error );
+			}
+
+			// Run one test URL per browser at a time
+			const browsersTaken = [];
+			for ( const test of queue ) {
+				if ( browsersTaken.indexOf( test.fullBrowser ) > -1 ) {
+					continue;
+				}
+				browsersTaken.push( test.fullBrowser );
+				if ( !test.running ) {
+					test.running = true;
+					try {
+						await createBrowserWorker( test.url, test.browser, test.options );
+					} catch ( error ) {
+						reject( error );
+					}
+				}
+			}
+			await new Promise( ( resolve ) => setTimeout( resolve, TEST_POLL_TIMEOUT ) );
+		}
+		resolve();
+	} );
+}
diff --git a/tests/runner/server.js b/tests/runner/server.js
new file mode 100644
index 000000000..10fbc220f
--- /dev/null
+++ b/tests/runner/server.js
@@ -0,0 +1,13 @@
+import { createTestServer } from "./createTestServer.js";
+
+const port = process.env.PORT || 3000;
+
+async function runServer() {
+	const app = await createTestServer();
+
+	app.listen( { port, host: "0.0.0.0" }, function() {
+		console.log( `Open tests at http://localhost:${ port }/tests/` );
+	} );
+}
+
+runServer();
diff --git a/tests/runner/suites.js b/tests/runner/suites.js
new file mode 100644
index 000000000..aa7732bf1
--- /dev/null
+++ b/tests/runner/suites.js
@@ -0,0 +1,26 @@
+export const suites = [
+	"accordion",
+	"autocomplete",
+	"button",
+	"checkboxradio",
+	"controlgroup",
+	"core",
+	"datepicker",
+	"dialog",
+	"draggable",
+	"droppable",
+	"effects",
+	"form-reset-mixin",
+	"menu",
+	"position",
+	"progressbar",
+	"resizable",
+	"selectable",
+	"selectmenu",
+	"slider",
+	"sortable",
+	"spinner",
+	"tabs",
+	"tooltip",
+	"widget"
+];
diff --git a/tests/unit/subsuite.js b/tests/unit/subsuite.js
index c34633a90..fa1533e51 100644
--- a/tests/unit/subsuite.js
+++ b/tests/unit/subsuite.js
@@ -17,7 +17,7 @@ var versions = [
 		"3.4.0", "3.4.1",
 		"3.5.0", "3.5.1",
 		"3.6.0", "3.6.1", "3.6.2", "3.6.3", "3.6.4",
-		"3.7.0",
+		"3.7.0", "3.7.1",
 		"3.x-git", "git", "custom"
 	],
 	additionalTests = {