aboutsummaryrefslogtreecommitdiffstats
path: root/test/runner
diff options
context:
space:
mode:
Diffstat (limited to 'test/runner')
-rw-r--r--test/runner/browsers.js25
-rw-r--r--test/runner/browserstack/api.js289
-rw-r--r--test/runner/browserstack/buildBrowserFromString.js10
-rw-r--r--test/runner/browserstack/createAuthHeader.js7
-rw-r--r--test/runner/browserstack/local.js34
-rw-r--r--test/runner/browserstack/workers.js276
-rw-r--r--test/runner/command.js120
-rw-r--r--test/runner/createTestServer.js60
-rw-r--r--test/runner/jsdom.js55
-rw-r--r--test/runner/lib/buildTestUrl.js29
-rw-r--r--test/runner/lib/generateHash.js50
-rw-r--r--test/runner/lib/getBrowserString.js49
-rw-r--r--test/runner/lib/prettyMs.js18
-rw-r--r--test/runner/listeners.js88
-rw-r--r--test/runner/modules.js24
-rw-r--r--test/runner/queue.js102
-rw-r--r--test/runner/reporter.js54
-rw-r--r--test/runner/run.js315
-rw-r--r--test/runner/selenium/createDriver.js80
-rw-r--r--test/runner/selenium/runSelenium.js31
-rw-r--r--test/runner/server.js13
21 files changed, 1729 insertions, 0 deletions
diff --git a/test/runner/browsers.js b/test/runner/browsers.js
new file mode 100644
index 000000000..a3a8df0d8
--- /dev/null
+++ b/test/runner/browsers.js
@@ -0,0 +1,25 @@
+// This list is static, so no requests are required
+// in the command help menu.
+
+import { getBrowsers } from "./browserstack/api.js";
+
+export const browsers = [
+ "chrome",
+ "ie",
+ "firefox",
+ "edge",
+ "safari",
+ "opera",
+ "yandex",
+ "IE Mobile",
+ "Android Browser",
+ "Mobile Safari",
+ "jsdom"
+];
+
+// A function that can be used to update the above list.
+export async function getAvailableBrowsers() {
+ const browsers = await getBrowsers( { flat: true } );
+ const available = [ ...new Set( browsers.map( ( { browser } ) => browser ) ) ];
+ return available.concat( "jsdom" );
+}
diff --git a/test/runner/browserstack/api.js b/test/runner/browserstack/api.js
new file mode 100644
index 000000000..40982c9b3
--- /dev/null
+++ b/test/runner/browserstack/api.js
@@ -0,0 +1,289 @@
+/**
+ * Browserstack API is documented at
+ * https://github.com/browserstack/api
+ */
+
+import { createAuthHeader } from "./createAuthHeader.js";
+
+const browserstackApi = "https://api.browserstack.com";
+const apiVersion = 5;
+
+const username = process.env.BROWSERSTACK_USERNAME;
+const accessKey = process.env.BROWSERSTACK_ACCESS_KEY;
+
+// iOS has null for version numbers,
+// and we do not need a similar check for OS versions.
+const rfinalVersion = /(?:^[0-9\.]+$)|(?:^null$)/;
+
+const rnonDigits = /(?:[^\d\.]+)|(?:20\d{2})/g;
+
+async function fetchAPI( path, options = {}, versioned = true ) {
+ if ( !username || !accessKey ) {
+ throw new Error(
+ "BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables must be set."
+ );
+ }
+ const init = {
+ method: "GET",
+ ...options,
+ headers: {
+ authorization: createAuthHeader( username, accessKey ),
+ accept: "application/json",
+ "content-type": "application/json",
+ ...options.headers
+ }
+ };
+ const response = await fetch(
+ `${ browserstackApi }/${ versioned ? `${ apiVersion }/` : "" }${ path }`,
+ init
+ );
+ if ( !response.ok ) {
+ console.log(
+ `\n${ init.method } ${ path }`,
+ response.status,
+ response.statusText
+ );
+ throw new Error( `Error fetching ${ path }` );
+ }
+ return response.json();
+}
+
+/**
+ * =============================
+ * Browsers API
+ * =============================
+ */
+
+function compareVersionNumbers( a, b ) {
+ if ( a != null && b == null ) {
+ return -1;
+ }
+ if ( a == null && b != null ) {
+ return 1;
+ }
+ if ( a == null && b == null ) {
+ return 0;
+ }
+ const aParts = a.replace( rnonDigits, "" ).split( "." );
+ const bParts = b.replace( rnonDigits, "" ).split( "." );
+
+ if ( aParts.length > bParts.length ) {
+ return -1;
+ }
+ if ( aParts.length < bParts.length ) {
+ return 1;
+ }
+
+ for ( let i = 0; i < aParts.length; i++ ) {
+ const aPart = Number( aParts[ i ] );
+ const bPart = Number( bParts[ i ] );
+ if ( aPart < bPart ) {
+ return -1;
+ }
+ if ( aPart > bPart ) {
+ return 1;
+ }
+ }
+}
+
+function sortBrowsers( a, b ) {
+ if ( a.browser < b.browser ) {
+ return -1;
+ }
+ if ( a.browser > b.browser ) {
+ return 1;
+ }
+ const browserComparison = compareVersionNumbers(
+ a.browser_version,
+ b.browser_version
+ );
+ if ( browserComparison ) {
+ return browserComparison;
+ }
+ if ( a.os < b.os ) {
+ return -1;
+ }
+ if ( a.os > b.os ) {
+ return 1;
+ }
+ const osComparison = compareVersionNumbers( a.os_version, b.os_version );
+ if ( osComparison ) {
+ return osComparison;
+ }
+ const deviceComparison = compareVersionNumbers( a.device, b.device );
+ if ( deviceComparison ) {
+ return deviceComparison;
+ }
+ return 0;
+}
+
+export async function getBrowsers( { flat = false } = {} ) {
+ const query = new URLSearchParams();
+ if ( flat ) {
+ query.append( "flat", true );
+ }
+ const browsers = await fetchAPI( `/browsers?${ query }` );
+ return browsers.sort( sortBrowsers );
+}
+
+function matchVersion( browserVersion, version ) {
+ if ( !version ) {
+ return false;
+ }
+ const regex = new RegExp(
+ `^${ version.replace( /\\/g, "\\\\" ).replace( /\./g, "\\." ) }\\b`,
+ "i"
+ );
+ return regex.test( browserVersion );
+}
+
+export async function filterBrowsers( filter ) {
+ const browsers = await getBrowsers( { flat: true } );
+ if ( !filter ) {
+ return browsers;
+ }
+ const filterBrowser = ( filter.browser ?? "" ).toLowerCase();
+ const filterVersion = ( filter.browser_version ?? "" ).toLowerCase();
+ const filterOs = ( filter.os ?? "" ).toLowerCase();
+ const filterOsVersion = ( filter.os_version ?? "" ).toLowerCase();
+ const filterDevice = ( filter.device ?? "" ).toLowerCase();
+
+ return browsers.filter( ( browser ) => {
+ return (
+ ( !filterBrowser || filterBrowser === browser.browser.toLowerCase() ) &&
+ ( !filterVersion ||
+ matchVersion( browser.browser_version, filterVersion ) ) &&
+ ( !filterOs || filterOs === browser.os.toLowerCase() ) &&
+ ( !filterOsVersion ||
+ filterOsVersion === browser.os_version.toLowerCase() ) &&
+ ( !filterDevice || filterDevice === ( browser.device || "" ).toLowerCase() )
+ );
+ } );
+}
+
+export async function listBrowsers( filter ) {
+ const browsers = await filterBrowsers( filter );
+ console.log( "Available browsers:" );
+ for ( const browser of browsers ) {
+ let message = ` ${ browser.browser }_`;
+ if ( browser.device ) {
+ message += `:${ browser.device }_`;
+ } else {
+ message += `${ browser.browser_version }_`;
+ }
+ message += `${ browser.os }_${ browser.os_version }`;
+ console.log( message );
+ }
+}
+
+export async function getLatestBrowser( filter ) {
+ const browsers = await filterBrowsers( filter );
+
+ // The list is sorted in ascending order,
+ // so the last item is the latest.
+ return browsers.findLast( ( browser ) =>
+ rfinalVersion.test( browser.browser_version )
+ );
+}
+
+/**
+ * =============================
+ * Workers API
+ * =============================
+ */
+
+/**
+ * A browser object may only have one of `browser` or `device` set;
+ * which property is set will depend on `os`.
+ *
+ * `options`: is an object with the following properties:
+ * `os`: The operating system.
+ * `os_version`: The operating system version.
+ * `browser`: The browser name.
+ * `browser_version`: The browser version.
+ * `device`: The device name.
+ * `url` (optional): Which URL to navigate to upon creation.
+ * `timeout` (optional): Maximum life of the worker (in seconds). Maximum value of `1800`. Specifying `0` will use the default of `300`.
+ * `name` (optional): Provide a name for the worker.
+ * `build` (optional): Group workers into a build.
+ * `project` (optional): Provide the project the worker belongs to.
+ * `resolution` (optional): Specify the screen resolution (e.g. "1024x768").
+ * `browserstack.local` (optional): Set to `true` to mark as local testing.
+ * `browserstack.video` (optional): Set to `false` to disable video recording.
+ * `browserstack.localIdentifier` (optional): ID of the local tunnel.
+ */
+export function createWorker( options ) {
+ return fetchAPI( "/worker", {
+ method: "POST",
+ body: JSON.stringify( options )
+ } );
+}
+
+/**
+ * Returns a worker object, if one exists, with the following properties:
+ * `id`: The worker id.
+ * `status`: A string representing the current status of the worker.
+ * Possible statuses: `"running"`, `"queue"`.
+ */
+export function getWorker( id ) {
+ return fetchAPI( `/worker/${ id }` );
+}
+
+export async function deleteWorker( id, verbose ) {
+ await fetchAPI( `/worker/${ id }`, { method: "DELETE" } );
+ if ( verbose ) {
+ console.log( `\nWorker ${ id } stopped.` );
+ }
+}
+
+export function getWorkers() {
+ return fetchAPI( "/workers" );
+}
+
+/**
+ * Change the URL of a worker,
+ * or refresh if it's the same URL.
+ */
+export function changeUrl( id, url ) {
+ return fetchAPI( `/worker/${ id }/url.json`, {
+ method: "PUT",
+ body: JSON.stringify( {
+ timeout: 20,
+ url: encodeURI( url )
+ } )
+ } );
+}
+
+/**
+ * Stop all workers
+ */
+export async function stopWorkers() {
+ const workers = await getWorkers();
+
+ // Run each request on its own
+ // to avoid connect timeout errors.
+ for ( const worker of workers ) {
+ try {
+ await deleteWorker( worker.id, true );
+ } catch ( error ) {
+
+ // Log the error, but continue trying to remove workers.
+ console.error( error );
+ }
+ }
+}
+
+/**
+ * =============================
+ * Plan API
+ * =============================
+ */
+
+export function getPlan() {
+ return fetchAPI( "/automate/plan.json", {}, false );
+}
+
+export async function getAvailableSessions() {
+ const [ plan, workers ] = await Promise.all( [ getPlan(), getWorkers() ] );
+ return plan.parallel_sessions_max_allowed - workers.length;
+}
diff --git a/test/runner/browserstack/buildBrowserFromString.js b/test/runner/browserstack/buildBrowserFromString.js
new file mode 100644
index 000000000..55aa38053
--- /dev/null
+++ b/test/runner/browserstack/buildBrowserFromString.js
@@ -0,0 +1,10 @@
+export function buildBrowserFromString( str ) {
+ const [ browser, versionOrDevice, os, osVersion ] = str.split( "_" );
+
+ // If the version starts with a colon, it's a device
+ if ( versionOrDevice && versionOrDevice.startsWith( ":" ) ) {
+ return { browser, device: versionOrDevice.slice( 1 ), os, os_version: osVersion };
+ }
+
+ return { browser, browser_version: versionOrDevice, os, os_version: osVersion };
+}
diff --git a/test/runner/browserstack/createAuthHeader.js b/test/runner/browserstack/createAuthHeader.js
new file mode 100644
index 000000000..fe4831e9a
--- /dev/null
+++ b/test/runner/browserstack/createAuthHeader.js
@@ -0,0 +1,7 @@
+const textEncoder = new TextEncoder();
+
+export function createAuthHeader( username, accessKey ) {
+ const encoded = textEncoder.encode( `${ username }:${ accessKey }` );
+ const base64 = btoa( String.fromCodePoint.apply( null, encoded ) );
+ return `Basic ${ base64 }`;
+}
diff --git a/test/runner/browserstack/local.js b/test/runner/browserstack/local.js
new file mode 100644
index 000000000..c84cf155c
--- /dev/null
+++ b/test/runner/browserstack/local.js
@@ -0,0 +1,34 @@
+import browserstackLocal from "browserstack-local";
+
+export async function localTunnel( localIdentifier, opts = {} ) {
+ const tunnel = new browserstackLocal.Local();
+
+ return new Promise( ( resolve, reject ) => {
+
+ // https://www.browserstack.com/docs/local-testing/binary-params
+ tunnel.start(
+ {
+ "enable-logging-for-api": "",
+ localIdentifier,
+ ...opts
+ },
+ async( error ) => {
+ if ( error || !tunnel.isRunning() ) {
+ return reject( error );
+ }
+ resolve( {
+ stop: function stopTunnel() {
+ return new Promise( ( resolve, reject ) => {
+ tunnel.stop( ( error ) => {
+ if ( error ) {
+ return reject( error );
+ }
+ resolve();
+ } );
+ } );
+ }
+ } );
+ }
+ );
+ } );
+}
diff --git a/test/runner/browserstack/workers.js b/test/runner/browserstack/workers.js
new file mode 100644
index 000000000..8f0ab68f0
--- /dev/null
+++ b/test/runner/browserstack/workers.js
@@ -0,0 +1,276 @@
+import chalk from "chalk";
+import { getBrowserString } from "../lib/getBrowserString.js";
+import { changeUrl, createWorker, deleteWorker, getWorker } from "./api.js";
+
+const workers = Object.create( null );
+
+// Acknowledge the worker within the time limit.
+// BrowserStack can take much longer spinning up
+// some browsers, such as iOS 15 Safari.
+const ACKNOWLEDGE_WORKER_TIMEOUT = 60 * 1000 * 8;
+const ACKNOWLEDGE_WORKER_INTERVAL = 1000;
+
+// No report after the time limit
+// should refresh the worker
+const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
+const MAX_WORKER_RESTARTS = 5;
+const MAX_WORKER_REFRESHES = 1;
+const POLL_WORKER_TIMEOUT = 1000;
+
+export async function cleanupWorker( reportId, verbose ) {
+ const worker = workers[ reportId ];
+ if ( worker ) {
+ try {
+ delete workers[ reportId ];
+ await deleteWorker( worker.id, verbose );
+ } catch ( error ) {
+ console.error( error );
+ }
+ }
+}
+
+export function debugWorker( reportId ) {
+ const worker = workers[ reportId ];
+ if ( worker ) {
+ worker.debug = true;
+ }
+}
+
+/**
+ * Set the last time a request was
+ * received related to the worker.
+ */
+export function touchWorker( reportId ) {
+ const worker = workers[ reportId ];
+ if ( worker ) {
+ worker.lastTouch = Date.now();
+ }
+}
+
+export function retryTest( reportId, retries ) {
+ const worker = workers[ reportId ];
+ if ( worker ) {
+ worker.retries ||= 0;
+ worker.retries++;
+ if ( worker.retries <= retries ) {
+ worker.retry = true;
+ console.log( `\nRetrying test ${ reportId }...${ worker.retries }` );
+ return true;
+ }
+ }
+ return false;
+}
+
+export async function cleanupAllWorkers( verbose ) {
+ const workersRemaining = Object.keys( workers ).length;
+ if ( workersRemaining ) {
+ if ( verbose ) {
+ console.log(
+ `Stopping ${ workersRemaining } stray worker${
+ workersRemaining > 1 ? "s" : ""
+ }...`
+ );
+ }
+ await Promise.all(
+ Object.values( workers ).map( ( worker ) => deleteWorker( worker.id, verbose ) )
+ );
+ }
+}
+
+async function waitForAck( id, verbose ) {
+ return new Promise( ( resolve, reject ) => {
+ const interval = setInterval( () => {
+ const worker = workers[ id ];
+ if ( !worker ) {
+ clearTimeout( timeout );
+ clearInterval( interval );
+ return reject( new Error( `Worker ${ id } not found.` ) );
+ }
+ if ( worker.lastTouch ) {
+ if ( verbose ) {
+ console.log( `\nWorker ${ id } acknowledged.` );
+ }
+ clearTimeout( timeout );
+ clearInterval( interval );
+ resolve();
+ }
+ }, ACKNOWLEDGE_WORKER_INTERVAL );
+ const timeout = setTimeout( () => {
+ clearInterval( interval );
+ const worker = workers[ id ];
+ reject(
+ new Error(
+ `Worker ${
+ worker ? worker.id : ""
+ } for test ${ id } not acknowledged after ${
+ ACKNOWLEDGE_WORKER_TIMEOUT / 1000
+ }s.`
+ )
+ );
+ }, ACKNOWLEDGE_WORKER_TIMEOUT );
+ } );
+}
+
+export async function runWorker(
+ url,
+ browser,
+ options,
+ restarts = 0
+) {
+ const { modules, reportId, runId, verbose } = options;
+ const worker = await createWorker( {
+ ...browser,
+ url: encodeURI( url ),
+ project: "jquery",
+ build: `Run ${ runId }`,
+ name: `${ modules.join( "," ) } (${ reportId })`,
+
+ // Set the max here, so that we can
+ // control the timeout
+ timeout: 1800,
+
+ // Not documented in the API docs,
+ // but required to make local testing work.
+ // See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
+ "browserstack.local": true,
+ "browserstack.localIdentifier": runId
+ } );
+
+ workers[ reportId ] = worker;
+
+ const timeMessage = `\nWorker ${
+ worker.id
+ } created for test ${ reportId } (${ chalk.yellow( getBrowserString( browser ) ) })`;
+
+ if ( verbose ) {
+ console.time( timeMessage );
+ }
+
+ async function retryWorker() {
+ await cleanupWorker( reportId, verbose );
+ if ( verbose ) {
+ console.log( `Retrying worker for test ${ reportId }...${ restarts + 1 }` );
+ }
+ return runWorker( url, browser, options, restarts + 1 );
+ }
+
+ // Wait for the worker to be acknowledged
+ try {
+ await waitForAck( reportId );
+ } catch ( error ) {
+ if ( !workers[ reportId ] ) {
+
+ // The worker has already been cleaned up
+ return;
+ }
+
+ if ( restarts < MAX_WORKER_RESTARTS ) {
+ return retryWorker();
+ }
+
+ throw error;
+ }
+
+ if ( verbose ) {
+ console.timeEnd( timeMessage );
+ }
+
+ let refreshes = 0;
+ let loggedStarted = false;
+ return new Promise( ( resolve, reject ) => {
+ async function refreshWorker() {
+ try {
+ await changeUrl( worker.id, url );
+ touchWorker( reportId );
+ return tick();
+ } catch ( error ) {
+ if ( !workers[ reportId ] ) {
+
+ // The worker has already been cleaned up
+ return resolve();
+ }
+ console.error( error );
+ return retryWorker().then( resolve, reject );
+ }
+ }
+
+ async function checkWorker() {
+ const worker = workers[ reportId ];
+
+ if ( !worker || worker.debug ) {
+ return resolve();
+ }
+
+ let fetchedWorker;
+ try {
+ fetchedWorker = await getWorker( worker.id );
+ } catch ( error ) {
+ return reject( error );
+ }
+ if (
+ !fetchedWorker ||
+ ( fetchedWorker.status !== "running" && fetchedWorker.status !== "queue" )
+ ) {
+ return resolve();
+ }
+
+ if ( verbose && !loggedStarted ) {
+ loggedStarted = true;
+ console.log(
+ `\nTest ${ chalk.bold( reportId ) } is ${
+ worker.status === "running" ? "running" : "in the queue"
+ }.`
+ );
+ console.log( ` View at ${ fetchedWorker.browser_url }.` );
+ }
+
+ // Refresh the worker if a retry is triggered
+ if ( worker.retry ) {
+ worker.retry = false;
+
+ // Reset recovery refreshes
+ refreshes = 0;
+ return refreshWorker();
+ }
+
+ if ( worker.lastTouch > Date.now() - RUN_WORKER_TIMEOUT ) {
+ return tick();
+ }
+
+ refreshes++;
+
+ if ( refreshes >= MAX_WORKER_REFRESHES ) {
+ if ( restarts < MAX_WORKER_RESTARTS ) {
+ if ( verbose ) {
+ console.log(
+ `Worker ${ worker.id } not acknowledged after ${
+ ACKNOWLEDGE_WORKER_TIMEOUT / 1000
+ }s.`
+ );
+ }
+ return retryWorker().then( resolve, reject );
+ }
+ await cleanupWorker( reportId, verbose );
+ return reject(
+ new Error(
+ `Worker ${ worker.id } for test ${ reportId } timed out after ${ MAX_WORKER_RESTARTS } restarts.`
+ )
+ );
+ }
+
+ if ( verbose ) {
+ console.log(
+ `\nRefreshing worker ${ worker.id } for test ${ reportId }...${ refreshes }`
+ );
+ }
+
+ return refreshWorker();
+ }
+
+ function tick() {
+ setTimeout( checkWorker, POLL_WORKER_TIMEOUT );
+ }
+
+ checkWorker();
+ } );
+}
diff --git a/test/runner/command.js b/test/runner/command.js
new file mode 100644
index 000000000..8419625a7
--- /dev/null
+++ b/test/runner/command.js
@@ -0,0 +1,120 @@
+import yargs from "yargs/yargs";
+import { browsers } from "./browsers.js";
+import { getPlan, listBrowsers, stopWorkers } from "./browserstack/api.js";
+import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js";
+import { modules } from "./modules.js";
+import { run } from "./run.js";
+
+const argv = yargs( process.argv.slice( 2 ) )
+ .version( false )
+ .command( {
+ command: "[options]",
+ describe: "Run jQuery tests in a browser"
+ } )
+ .option( "module", {
+ alias: "m",
+ type: "array",
+ choices: modules,
+ description:
+ "Run tests for a specific module. " +
+ "Pass multiple modules by repeating the option. " +
+ "Defaults to all modules."
+ } )
+ .option( "browser", {
+ alias: "b",
+ type: "array",
+ choices: browsers,
+ description:
+ "Run tests in a specific browser." +
+ "Pass multiple browsers by repeating the option." +
+ "If using BrowserStack, specify browsers using --browserstack." +
+ "Only the basic module is supported on jsdom.",
+ default: [ "chrome" ]
+ } )
+ .option( "headless", {
+ alias: "h",
+ type: "boolean",
+ description:
+ "Run tests in headless mode. Cannot be used with --debug or --browserstack.",
+ conflicts: [ "debug", "browserstack" ]
+ } )
+ .option( "esm", {
+ alias: "esmodules",
+ type: "boolean",
+ description: "Run tests using jQuery's source, which is written with ECMAScript Modules."
+ } )
+ .option( "concurrency", {
+ alias: "c",
+ type: "number",
+ description:
+ "Run tests in parallel in multiple browsers. " +
+ "Defaults to 8 in normal mode. In browserstack mode, defaults to the maximum available under your BrowserStack plan."
+ } )
+ .option( "debug", {
+ alias: "d",
+ type: "boolean",
+ description:
+ "Leave the browser open for debugging. Cannot be used with --headless.",
+ conflicts: [ "headless" ]
+ } )
+ .option( "verbose", {
+ alias: "v",
+ type: "boolean",
+ description: "Log additional information."
+ } )
+ .option( "retries", {
+ alias: "r",
+ type: "number",
+ description: "Number of times to retry failed tests.",
+ default: 0
+ } )
+ .option( "no-isolate", {
+ type: "boolean",
+ description: "Run all modules in the same browser instance."
+ } )
+ .option( "browserstack", {
+ type: "array",
+ description:
+ "Run tests in BrowserStack.\nRequires BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables.\n" +
+ "The value can be empty for the default configuration, or a string in the format of\n" +
+ "\"browser_[browserVersion | :device]_os_osVersion\" (see --list-browsers).\n" +
+ "Pass multiple browsers by repeating the option. The --browser option is ignored when --browserstack has a value.\n" +
+ "Otherwise, the --browser option will be used, with the latest version/device for that browser, on a matching OS."
+ } )
+ .option( "list-browsers", {
+ type: "string",
+ description:
+ "List available BrowserStack browsers and exit.\n" +
+ "Leave blank to view all browsers or pass " +
+ "\"browser_[browserVersion | :device]_os_osVersion\" with each parameter " +
+ "separated by an underscore to filter the list (any can be omitted).\n" +
+ "Use a colon to indicate a device.\n" +
+ "Examples: \"chrome__windows_10\", \"Mobile Safari\", \"Android Browser_:Google Pixel 8 Pro\".\n" +
+ "Use quotes if spaces are necessary."
+ } )
+ .option( "stop-workers", {
+ type: "boolean",
+ description:
+ "WARNING: This will stop all BrowserStack workers that may exist and exit," +
+ "including any workers running from other projects.\n" +
+ "This can be used as a failsafe when there are too many stray workers."
+ } )
+ .option( "browserstack-plan", {
+ type: "boolean",
+ description: "Show BrowserStack plan information and exit."
+ } )
+ .help().argv;
+
+if ( typeof argv.listBrowsers === "string" ) {
+ listBrowsers( buildBrowserFromString( argv.listBrowsers ) );
+} else if ( argv.stopWorkers ) {
+ stopWorkers();
+} else if ( argv.browserstackPlan ) {
+ console.log( await getPlan() );
+} else {
+ run( {
+ ...argv,
+ browsers: argv.browser,
+ modules: argv.module
+ } );
+}
diff --git a/test/runner/createTestServer.js b/test/runner/createTestServer.js
new file mode 100644
index 000000000..ebc6bd4bb
--- /dev/null
+++ b/test/runner/createTestServer.js
@@ -0,0 +1,60 @@
+import bodyParser from "body-parser";
+import express from "express";
+import bodyParserErrorHandler from "express-body-parser-error-handler";
+import fs from "fs";
+import mockServer from "../middleware-mockserver.cjs";
+
+export async function createTestServer( report ) {
+ const nameHTML = await fs.promises.readFile( "./test/data/name.html", "utf8" );
+ const indexHTML = await fs.promises.readFile( "./test/index.html", "utf8" );
+ const app = express();
+
+ // Redirect home to test page
+ app.get( "/", ( _req, res ) => {
+ res.redirect( "/test/" );
+ } );
+
+ // Redirect to trailing slash
+ app.use( ( req, res, next ) => {
+ if ( req.path === "/test" ) {
+ const query = req.url.slice( req.path.length );
+ res.redirect( 301, `${ req.path }/${ query }` );
+ } else {
+ next();
+ }
+ } );
+
+ // Add a script tag to the index.html to load the QUnit listeners
+ app.use( /\/test(?:\/index.html)?\//, ( _req, res ) => {
+ res.send( indexHTML.replace(
+ "</head>",
+ "<script src=\"/test/runner/listeners.js\"></script></head>"
+ ) );
+ } );
+
+ // Bind the reporter
+ app.post( "/api/report", bodyParser.json( { limit: "50mb" } ), ( req, res ) => {
+ if ( report ) {
+ report( req.body );
+ }
+ res.sendStatus( 204 );
+ } );
+
+ // Handle errors from the body parser
+ app.use( bodyParserErrorHandler() );
+
+ // Hook up mock server
+ app.use( mockServer() );
+
+ // Serve static files
+ app.post( "/test/data/name.html", ( _req, res ) => {
+ res.send( nameHTML );
+ } );
+
+ app.use( "/dist", express.static( "dist" ) );
+ app.use( "/src", express.static( "src" ) );
+ app.use( "/test", express.static( "test" ) );
+ app.use( "/external", express.static( "external" ) );
+
+ return app;
+}
diff --git a/test/runner/jsdom.js b/test/runner/jsdom.js
new file mode 100644
index 000000000..d370ac348
--- /dev/null
+++ b/test/runner/jsdom.js
@@ -0,0 +1,55 @@
+import jsdom from "jsdom";
+
+const { JSDOM } = jsdom;
+
+const windows = Object.create( null );
+
+export async function runJSDOM( url, { reportId, verbose } ) {
+ const virtualConsole = new jsdom.VirtualConsole();
+ virtualConsole.sendTo( console );
+ virtualConsole.removeAllListeners( "clear" );
+
+ const { window } = await JSDOM.fromURL( url, {
+ resources: "usable",
+ runScripts: "dangerously",
+ virtualConsole
+ } );
+ if ( verbose ) {
+ console.log( "JSDOM window opened.", reportId );
+ }
+ windows[ reportId ] = window;
+
+ return new Promise( ( resolve ) => {
+ window.finish = resolve;
+ } );
+}
+
+export function cleanupJSDOM( reportId, verbose ) {
+ const window = windows[ reportId ];
+ if ( window ) {
+ if ( window.finish ) {
+ window.finish();
+ }
+ window.close();
+ delete windows[ reportId ];
+ if ( verbose ) {
+ console.log( "Closed JSDOM window.", reportId );
+ }
+ }
+}
+
+export function cleanupAllJSDOM( verbose ) {
+ const windowsRemaining = Object.keys( windows ).length;
+ if ( windowsRemaining ) {
+ if ( verbose ) {
+ console.log(
+ `Cleaning up ${ windowsRemaining } JSDOM window${
+ windowsRemaining > 1 ? "s" : ""
+ }...`
+ );
+ }
+ for ( const id in windows ) {
+ cleanupJSDOM( id, verbose );
+ }
+ }
+}
diff --git a/test/runner/lib/buildTestUrl.js b/test/runner/lib/buildTestUrl.js
new file mode 100644
index 000000000..6e0f1a9b0
--- /dev/null
+++ b/test/runner/lib/buildTestUrl.js
@@ -0,0 +1,29 @@
+import { generateModuleId } from "./generateHash.js";
+
+export function buildTestUrl( modules, { browserstack, esm, jsdom, port, reportId } ) {
+ if ( !port ) {
+ throw new Error( "No port specified." );
+ }
+
+ const query = new URLSearchParams();
+ for ( const module of modules ) {
+ query.append( "moduleId", generateModuleId( module ) );
+ }
+
+ if ( esm ) {
+ query.append( "esmodules", "true" );
+ }
+
+ if ( jsdom ) {
+ query.append( "jsdom", "true" );
+ }
+
+ if ( reportId ) {
+ query.append( "reportId", reportId );
+ }
+
+ // BrowserStack supplies a custom domain for local testing,
+ // which is especially necessary for iOS testing.
+ const host = browserstack ? "bs-local.com" : "localhost";
+ return `http://${ host }:${ port }/test/?${ query }`;
+}
diff --git a/test/runner/lib/generateHash.js b/test/runner/lib/generateHash.js
new file mode 100644
index 000000000..dbbd4b476
--- /dev/null
+++ b/test/runner/lib/generateHash.js
@@ -0,0 +1,50 @@
+import crypto from "node:crypto";
+
+export function generateHash( string ) {
+ const hash = crypto.createHash( "md5" );
+ hash.update( string );
+
+ // QUnit hashes are 8 characters long
+ // We use 10 characters to be more visually distinct
+ return hash.digest( "hex" ).slice( 0, 10 );
+}
+
+/**
+ * A copy of the generate hash function from QUnit,
+ * used to generate a hash for the module name.
+ *
+ * QUnit errors on passing multiple modules to the
+ * module query parameter. We need to know
+ * the hash for each module before loading QUnit
+ * in order to pass multiple moduleId parameters instead.
+ */
+export function generateModuleId( module, browser ) {
+
+ // QUnit normally hashes the test name, but
+ // we've repurposed this function to generate
+ // report IDs for module/browser combinations.
+ // We still use it without the browser parameter
+ // to get the same module IDs as QUnit to pass
+ // multiple ahead-of-time in the query string.
+ const str = module + "\x1C" + browser;
+ let hash = 0;
+
+ for ( let i = 0; i < str.length; i++ ) {
+ hash = ( hash << 5 ) - hash + str.charCodeAt( i );
+ hash |= 0;
+ }
+
+ let hex = ( 0x100000000 + hash ).toString( 16 );
+ if ( hex.length < 8 ) {
+ hex = "0000000" + hex;
+ }
+
+ return hex.slice( -8 );
+}
+
+export function printModuleHashes( modules ) {
+ console.log( "Module hashes:" );
+ modules.forEach( ( module ) => {
+ console.log( ` ${ module }: ${ generateModuleId( module ) }` );
+ } );
+}
diff --git a/test/runner/lib/getBrowserString.js b/test/runner/lib/getBrowserString.js
new file mode 100644
index 000000000..413a60500
--- /dev/null
+++ b/test/runner/lib/getBrowserString.js
@@ -0,0 +1,49 @@
+const browserMap = {
+ chrome: "Chrome",
+ edge: "Edge",
+ firefox: "Firefox",
+ ie: "IE",
+ jsdom: "JSDOM",
+ opera: "Opera",
+ safari: "Safari"
+};
+
+export function browserSupportsHeadless( browser ) {
+ browser = browser.toLowerCase();
+ return (
+ browser === "chrome" ||
+ browser === "firefox" ||
+ browser === "edge"
+ );
+}
+
+export function getBrowserString(
+ {
+ browser,
+ browser_version: browserVersion,
+ device,
+ os,
+ os_version: osVersion
+ },
+ headless
+) {
+ browser = browser.toLowerCase();
+ browser = browserMap[ browser ] || browser;
+ let str = browser;
+ if ( browserVersion ) {
+ str += ` ${ browserVersion }`;
+ }
+ if ( device ) {
+ str += ` for ${ device }`;
+ }
+ if ( os ) {
+ str += ` on ${ os }`;
+ }
+ if ( osVersion ) {
+ str += ` ${ osVersion }`;
+ }
+ if ( headless && browserSupportsHeadless( browser ) ) {
+ str += " (headless)";
+ }
+ return str;
+}
diff --git a/test/runner/lib/prettyMs.js b/test/runner/lib/prettyMs.js
new file mode 100644
index 000000000..99bae2b35
--- /dev/null
+++ b/test/runner/lib/prettyMs.js
@@ -0,0 +1,18 @@
+/**
+ * Pretty print a time in milliseconds.
+ */
+export function prettyMs( time ) {
+ const minutes = Math.floor( time / 60000 );
+ const seconds = Math.floor( time / 1000 );
+ const ms = Math.floor( time % 1000 );
+
+ let prettyTime = `${ ms }ms`;
+ if ( seconds > 0 ) {
+ prettyTime = `${ seconds }s ${ prettyTime }`;
+ }
+ if ( minutes > 0 ) {
+ prettyTime = `${ minutes }m ${ prettyTime }`;
+ }
+
+ return prettyTime;
+}
diff --git a/test/runner/listeners.js b/test/runner/listeners.js
new file mode 100644
index 000000000..a3c52c21e
--- /dev/null
+++ b/test/runner/listeners.js
@@ -0,0 +1,88 @@
+( function() {
+
+ "use strict";
+
+ // Get the report ID from the URL.
+ var match = location.search.match( /reportId=([^&]+)/ );
+ if ( !match ) {
+ return;
+ }
+ var id = match[ 1 ];
+
+ // Adopted from https://github.com/douglascrockford/JSON-js
+ // Support: IE 11+
+ // Using the replacer argument of JSON.stringify in IE has issues
+ // TODO: Replace this with a circular replacer + JSON.stringify + WeakSet
+ function decycle( object ) {
+ var objects = [];
+
+ // The derez function recurses through the object, producing the deep copy.
+ function derez( value ) {
+ if (
+ typeof value === "object" &&
+ value !== null &&
+ !( value instanceof Boolean ) &&
+ !( value instanceof Date ) &&
+ !( value instanceof Number ) &&
+ !( value instanceof RegExp ) &&
+ !( value instanceof String )
+ ) {
+
+ // Return a string early for elements
+ if ( value.nodeType ) {
+ return value.toString();
+ }
+
+ if ( objects.indexOf( value ) > -1 ) {
+ return;
+ }
+
+ objects.push( value );
+
+ if ( Array.isArray( value ) ) {
+
+ // If it is an array, replicate the array.
+ return value.map( derez );
+ } else {
+
+ // If it is an object, replicate the object.
+ var nu = Object.create( null );
+ Object.keys( value ).forEach( function( name ) {
+ nu[ name ] = derez( value[ name ] );
+ } );
+ return nu;
+ }
+ }
+ return value;
+ }
+ return derez( object );
+ }
+
+ function send( type, data ) {
+ var json = JSON.stringify( {
+ id: id,
+ type: type,
+ data: data ? decycle( data ) : undefined
+ } );
+ var request = new XMLHttpRequest();
+ request.open( "POST", "/api/report", true );
+ request.setRequestHeader( "Content-Type", "application/json" );
+ request.send( json );
+ }
+
+ // Send acknowledgement to the server.
+ send( "ack" );
+
+ QUnit.on( "testEnd", function( data ) {
+ send( "testEnd", data );
+ } );
+
+ QUnit.on( "runEnd", function( data ) {
+
+ // Reduce the payload size.
+ // childSuites is large and unused.
+ data.childSuites = undefined;
+
+ send( "runEnd", data );
+ } );
+} )();
diff --git a/test/runner/modules.js b/test/runner/modules.js
new file mode 100644
index 000000000..53f9a933a
--- /dev/null
+++ b/test/runner/modules.js
@@ -0,0 +1,24 @@
+export const modules = [
+ "basic",
+
+ "ajax",
+ "animation",
+ "attributes",
+ "callbacks",
+ "core",
+ "css",
+ "data",
+ "deferred",
+ "deprecated",
+ "dimensions",
+ "effects",
+ "event",
+ "manipulation",
+ "offset",
+ "queue",
+ "selector",
+ "serialize",
+ "support",
+ "traversing",
+ "tween"
+];
diff --git a/test/runner/queue.js b/test/runner/queue.js
new file mode 100644
index 000000000..4c9d66d8f
--- /dev/null
+++ b/test/runner/queue.js
@@ -0,0 +1,102 @@
+// Build a queue that runs both browsers and modules
+// in parallel when the length reaches the concurrency limit
+// and refills the queue when one promise resolves.
+
+import chalk from "chalk";
+import { getAvailableSessions } from "./browserstack/api.js";
+import { runWorker } from "./browserstack/workers.js";
+import { getBrowserString } from "./lib/getBrowserString.js";
+import { runSelenium } from "./selenium/runSelenium.js";
+import { runJSDOM } from "./jsdom.js";
+
+const queue = [];
+const promises = [];
+
+const SELENIUM_WAIT_TIME = 100;
+const BROWSERSTACK_WAIT_TIME = 5000;
+const WORKER_WAIT_TIME = 30000;
+
+// Limit concurrency to 8 by default in selenium
+// BrowserStack defaults to the max allowed by the plan
+// More than this will log MaxListenersExceededWarning
+const MAX_CONCURRENCY = 8;
+
+export function addRun( url, browser, options ) {
+ queue.push( { url, browser, options } );
+}
+
+export async function runFullQueue( {
+ browserstack,
+ concurrency: defaultConcurrency,
+ verbose
+} ) {
+ while ( queue.length ) {
+ const next = queue.shift();
+ const { url, browser, options } = next;
+
+ const fullBrowser = getBrowserString( browser, options.headless );
+ console.log(
+ `\nRunning ${ chalk.yellow( options.modules.join( ", " ) ) } tests ` +
+ `in ${ chalk.yellow( fullBrowser ) } (${ chalk.bold( options.reportId ) })...`
+ );
+
+ // Wait enough time between requests
+ // to give concurrency a chance to update.
+ // In selenium, this helps avoid undici connect timeout errors.
+ await new Promise( ( resolve ) =>
+ setTimeout(
+ resolve,
+ browserstack ? BROWSERSTACK_WAIT_TIME : SELENIUM_WAIT_TIME
+ )
+ );
+
+ const concurrency =
+ browserstack && !defaultConcurrency ?
+ await getAvailableSessions() :
+ defaultConcurrency || MAX_CONCURRENCY;
+
+ if ( verbose ) {
+ console.log(
+ `\nConcurrency: ${ concurrency }. Tests remaining: ${ queue.length + 1 }.`
+ );
+ }
+
+ // If concurrency is 0, wait a bit and try again
+ if ( concurrency <= 0 ) {
+ if ( verbose ) {
+ console.log( "\nWaiting for available sessions..." );
+ }
+ queue.unshift( next );
+ await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
+ continue;
+ }
+
+ let promise;
+ if ( browser.browser === "jsdom" ) {
+ promise = runJSDOM( url, options );
+ } else if ( browserstack ) {
+ promise = runWorker( url, browser, options );
+ } else {
+ promise = runSelenium( url, browser, options );
+ }
+
+ // Remove the promise from the list when it resolves
+ promise.then( () => {
+ const index = promises.indexOf( promise );
+ if ( index !== -1 ) {
+ promises.splice( index, 1 );
+ }
+ } );
+
+ // Add the promise to the list
+ promises.push( promise );
+
+ // Wait until at least one promise resolves
+ // if we've reached the concurrency limit
+ if ( promises.length >= concurrency ) {
+ await Promise.any( promises );
+ }
+ }
+
+ await Promise.all( promises );
+}
diff --git a/test/runner/reporter.js b/test/runner/reporter.js
new file mode 100644
index 000000000..bb5e7601b
--- /dev/null
+++ b/test/runner/reporter.js
@@ -0,0 +1,54 @@
+import chalk from "chalk";
+import { getBrowserString } from "./lib/getBrowserString.js";
+import { prettyMs } from "./lib/prettyMs.js";
+
+export function reportTest( test, reportId, { browser, headless } ) {
+ if ( test.status === "passed" ) {
+
+ // Write to console without newlines
+ process.stdout.write( "." );
+ return;
+ }
+
+ let message = `Test ${ test.status } on ${ chalk.yellow(
+ getBrowserString( browser, headless )
+ ) } (${ chalk.bold( reportId ) }).`;
+ message += `\n${ chalk.bold( `${ test.suiteName }: ${ test.name }` ) }`;
+
+ // Prefer failed assertions over error messages
+ if ( test.assertions.filter( ( a ) => !!a && !a.passed ).length ) {
+ test.assertions.forEach( ( assertion, i ) => {
+ if ( !assertion.passed ) {
+ message += `\n${ i + 1 }. ${ chalk.red( assertion.message ) }`;
+ message += `\n${ chalk.gray( assertion.stack ) }`;
+ }
+ } );
+ } else if ( test.errors.length ) {
+ for ( const error of test.errors ) {
+ message += `\n${ chalk.red( error.message ) }`;
+ message += `\n${ chalk.gray( error.stack ) }`;
+ }
+ }
+
+ console.log( "\n\n" + message );
+
+ if ( test.status === "failed" ) {
+ return message;
+ }
+}
+
+export function reportEnd( result, reportId, { browser, headless, modules } ) {
+ console.log(
+ `\n\nTests for ${ chalk.yellow( modules.join( ", " ) ) } on ${ chalk.yellow(
+ getBrowserString( browser, headless )
+ ) } finished in ${ prettyMs( result.runtime ) } (${ chalk.bold( reportId ) }).`
+ );
+ console.log(
+ ( result.status !== "passed" ?
+ `${ chalk.red( result.testCounts.failed ) } failed. ` :
+ "" ) +
+ `${ chalk.green( result.testCounts.total ) } passed. ` +
+ `${ chalk.gray( result.testCounts.skipped ) } skipped.`
+ );
+ return result.testCounts;
+}
diff --git a/test/runner/run.js b/test/runner/run.js
new file mode 100644
index 000000000..89b0c5eb7
--- /dev/null
+++ b/test/runner/run.js
@@ -0,0 +1,315 @@
+import chalk from "chalk";
+import { asyncExitHook, gracefulExit } from "exit-hook";
+import { getLatestBrowser } from "./browserstack/api.js";
+import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js";
+import { localTunnel } from "./browserstack/local.js";
+import { reportEnd, reportTest } from "./reporter.js";
+import {
+ cleanupAllWorkers,
+ cleanupWorker,
+ debugWorker,
+ retryTest,
+ touchWorker
+} from "./browserstack/workers.js";
+import { createTestServer } from "./createTestServer.js";
+import { buildTestUrl } from "./lib/buildTestUrl.js";
+import { generateHash, printModuleHashes } from "./lib/generateHash.js";
+import { getBrowserString } from "./lib/getBrowserString.js";
+import { addRun, runFullQueue } from "./queue.js";
+import { cleanupAllJSDOM, cleanupJSDOM } from "./jsdom.js";
+import { modules as allModules } from "./modules.js";
+
+const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
+
+/**
+ * Run modules in parallel in different browser instances.
+ */
+export async function run( {
+ browsers: browserNames,
+ browserstack,
+ concurrency,
+ debug,
+ esm,
+ headless,
+ isolate = true,
+ modules = [],
+ retries = 3,
+ verbose
+} = {} ) {
+ if ( !browserNames || !browserNames.length ) {
+ browserNames = [ "chrome" ];
+ }
+ if ( !modules.length ) {
+ modules = allModules;
+ }
+ if ( headless && debug ) {
+ throw new Error(
+ "Cannot run in headless mode and debug mode at the same time."
+ );
+ }
+
+ if ( verbose ) {
+ console.log( browserstack ? "Running in BrowserStack." : "Running locally." );
+ }
+
+ const errorMessages = [];
+ const pendingErrors = {};
+
+ // Convert browser names to browser objects
+ let browsers = browserNames.map( ( b ) => ( { browser: b } ) );
+
+ // A unique identifier for this run
+ const runId = generateHash(
+ `${ Date.now() }-${ modules.join( ":" ) }-${ ( browserstack || [] )
+ .concat( browserNames )
+ .join( ":" ) }`
+ );
+
+ // Create the test app and
+ // hook it up to the reporter
+ const reports = Object.create( null );
+ const app = await createTestServer( async( message ) => {
+ switch ( message.type ) {
+ case "testEnd": {
+ const reportId = message.id;
+ touchWorker( reportId );
+ const errors = reportTest( message.data, reportId, reports[ reportId ] );
+ pendingErrors[ reportId ] ||= {};
+ if ( errors ) {
+ pendingErrors[ reportId ][ message.data.name ] = errors;
+ } else {
+ delete pendingErrors[ reportId ][ message.data.name ];
+ }
+ break;
+ }
+ case "runEnd": {
+ const reportId = message.id;
+ const report = reports[ reportId ];
+ const { failed, total } = reportEnd(
+ message.data,
+ message.id,
+ reports[ reportId ]
+ );
+ report.total = total;
+
+ if ( failed ) {
+ if ( !retryTest( reportId, retries ) ) {
+ if ( debug ) {
+ debugWorker( reportId );
+ }
+ errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
+ }
+ } else {
+ if ( Object.keys( pendingErrors[ reportId ] ).length ) {
+ console.warn( "Detected flaky tests:" );
+ for ( const [ , error ] in Object.entries( pendingErrors[ reportId ] ) ) {
+ console.warn( chalk.italic( chalk.gray( error ) ) );
+ }
+ delete pendingErrors[ reportId ];
+ }
+ }
+ await cleanupWorker( reportId, verbose );
+ cleanupJSDOM( reportId, verbose );
+ break;
+ }
+ case "ack": {
+ touchWorker( message.id );
+ if ( verbose ) {
+ console.log( `\nWorker for test ${ message.id } acknowledged.` );
+ }
+ break;
+ }
+ default:
+ console.warn( "Received unknown message type:", message.type );
+ }
+ } );
+
+ // Start up local test server
+ let server;
+ let port;
+ await new Promise( ( resolve ) => {
+
+ // Pass 0 to choose a random, unused port
+ server = app.listen( 0, () => {
+ port = server.address().port;
+ resolve();
+ } );
+ } );
+
+ if ( !server || !port ) {
+ throw new Error( "Server not started." );
+ }
+
+ if ( verbose ) {
+ console.log( `Server started on port ${ port }.` );
+ }
+
+ function stopServer() {
+ return new Promise( ( resolve ) => {
+ server.close( () => {
+ if ( verbose ) {
+ console.log( "Server stopped." );
+ }
+ resolve();
+ } );
+ } );
+ }
+
+ async function cleanup() {
+ console.log( "Cleaning up..." );
+
+ if ( tunnel ) {
+ await tunnel.stop();
+ if ( verbose ) {
+ console.log( "Stopped BrowserStackLocal." );
+ }
+ }
+
+ await cleanupAllWorkers( verbose );
+ cleanupAllJSDOM( verbose );
+ }
+
+ asyncExitHook(
+ async() => {
+ await stopServer();
+ await cleanup();
+ },
+ { wait: EXIT_HOOK_WAIT_TIMEOUT }
+ );
+
+ // Start up BrowserStackLocal
+ let tunnel;
+ if ( browserstack ) {
+ if ( headless ) {
+ console.warn(
+ chalk.italic(
+ "BrowserStack does not support headless mode. Running in normal mode."
+ )
+ );
+ headless = false;
+ }
+
+ // Convert browserstack to browser objects.
+ // If browserstack is an empty array, fall back
+ // to the browsers array.
+ if ( browserstack.length ) {
+ browsers = browserstack.map( ( b ) => {
+ if ( !b ) {
+ return browsers[ 0 ];
+ }
+ return buildBrowserFromString( b );
+ } );
+ }
+
+ // Fill out browser defaults
+ browsers = await Promise.all(
+ browsers.map( async( browser ) => {
+
+ // Avoid undici connect timeout errors
+ await new Promise( ( resolve ) => setTimeout( resolve, 100 ) );
+
+ const latestMatch = await getLatestBrowser( browser );
+ if ( !latestMatch ) {
+ throw new Error( `Browser not found: ${ getBrowserString( browser ) }.` );
+ }
+ return latestMatch;
+ } )
+ );
+
+ tunnel = await localTunnel( runId );
+ if ( verbose ) {
+ console.log( "Started BrowserStackLocal." );
+
+ printModuleHashes( modules );
+ }
+ }
+
+ function queueRun( modules, browser ) {
+ const fullBrowser = getBrowserString( browser, headless );
+ const reportId = generateHash( `${ modules.join( ":" ) } ${ fullBrowser }` );
+ reports[ reportId ] = { browser, headless, modules };
+
+ const url = buildTestUrl( modules, {
+ browserstack,
+ esm,
+ jsdom: browser.browser === "jsdom",
+ port,
+ reportId
+ } );
+
+ addRun( url, browser, {
+ debug,
+ headless,
+ modules,
+ reportId,
+ retries,
+ runId,
+ verbose
+ } );
+ }
+
+ for ( const browser of browsers ) {
+ if ( isolate ) {
+ for ( const module of modules ) {
+ queueRun( [ module ], browser );
+ }
+ } else {
+ queueRun( modules, browser );
+ }
+ }
+
+ try {
+ console.log( `Starting Run ${ runId }...` );
+ await runFullQueue( { browserstack, concurrency, verbose } );
+ } catch ( error ) {
+ console.error( error );
+ if ( !debug ) {
+ gracefulExit( 1 );
+ }
+ } finally {
+ console.log();
+ if ( errorMessages.length === 0 ) {
+ let stop = false;
+ for ( const report of Object.values( reports ) ) {
+ if ( !report.total ) {
+ stop = true;
+ console.error(
+ chalk.red(
+ `No tests were run for ${ report.modules.join(
+ ", "
+ ) } in ${ getBrowserString( report.browser ) }`
+ )
+ );
+ }
+ }
+ if ( stop ) {
+ return gracefulExit( 1 );
+ }
+ console.log( chalk.green( "All tests passed!" ) );
+
+ if ( !debug || browserstack ) {
+ gracefulExit( 0 );
+ }
+ } else {
+ console.error( chalk.red( `${ errorMessages.length } tests failed.` ) );
+ console.log(
+ errorMessages.map( ( error, i ) => `\n${ i + 1 }. ${ error }` ).join( "\n" )
+ );
+
+ if ( debug ) {
+ console.log();
+ if ( browserstack ) {
+ console.log( "Leaving browsers with failures open for debugging." );
+ console.log(
+ "View running sessions at https://automate.browserstack.com/dashboard/v2/"
+ );
+ } else {
+ console.log( "Leaving browsers open for debugging." );
+ }
+ console.log( "Press Ctrl+C to exit." );
+ } else {
+ gracefulExit( 1 );
+ }
+ }
+ }
+}
diff --git a/test/runner/selenium/createDriver.js b/test/runner/selenium/createDriver.js
new file mode 100644
index 000000000..765ebb847
--- /dev/null
+++ b/test/runner/selenium/createDriver.js
@@ -0,0 +1,80 @@
+import { Builder, Capabilities, logging } from "selenium-webdriver";
+import Chrome from "selenium-webdriver/chrome.js";
+import Edge from "selenium-webdriver/edge.js";
+import Firefox from "selenium-webdriver/firefox.js";
+import { browserSupportsHeadless } from "../lib/getBrowserString.js";
+
+// Set script timeout to 10min
+const DRIVER_SCRIPT_TIMEOUT = 1000 * 60 * 10;
+
+export default async function createDriver( { browserName, headless, verbose } ) {
+ const capabilities = Capabilities[ browserName ]();
+ const prefs = new logging.Preferences();
+ prefs.setLevel( logging.Type.BROWSER, logging.Level.ALL );
+ capabilities.setLoggingPrefs( prefs );
+
+ let driver = new Builder().withCapabilities( capabilities );
+
+ const chromeOptions = new Chrome.Options();
+ chromeOptions.addArguments( "--enable-chrome-browser-cloud-management" );
+
+ // Alter the chrome binary path if
+ // the CHROME_BIN environment variable is set
+ if ( process.env.CHROME_BIN ) {
+ if ( verbose ) {
+ console.log( `Setting chrome binary to ${ process.env.CHROME_BIN }` );
+ }
+ chromeOptions.setChromeBinaryPath( process.env.CHROME_BIN );
+ }
+
+ const firefoxOptions = new Firefox.Options();
+
+ if ( process.env.FIREFOX_BIN ) {
+ if ( verbose ) {
+ console.log( `Setting firefox binary to ${ process.env.FIREFOX_BIN }` );
+ }
+
+ firefoxOptions.setBinary( process.env.FIREFOX_BIN );
+ }
+
+ const edgeOptions = new Edge.Options();
+ edgeOptions.addArguments( "--enable-chrome-browser-cloud-management" );
+
+ // Alter the edge binary path if
+ // the EDGE_BIN environment variable is set
+ if ( process.env.EDGE_BIN ) {
+ if ( verbose ) {
+ console.log( `Setting edge binary to ${ process.env.EDGE_BIN }` );
+ }
+ edgeOptions.setEdgeChromiumBinaryPath( process.env.EDGE_BIN );
+ }
+
+ if ( headless ) {
+ chromeOptions.addArguments( "--headless=new" );
+ firefoxOptions.addArguments( "--headless" );
+ edgeOptions.addArguments( "--headless=new" );
+ if ( !browserSupportsHeadless( browserName ) ) {
+ console.log(
+ `Headless mode is not supported for ${ browserName }. Running in normal mode instead.`
+ );
+ }
+ }
+
+ driver = await driver
+ .setChromeOptions( chromeOptions )
+ .setFirefoxOptions( firefoxOptions )
+ .setEdgeOptions( edgeOptions )
+ .build();
+
+ if ( verbose ) {
+ const driverCapabilities = await driver.getCapabilities();
+ const name = driverCapabilities.getBrowserName();
+ const version = driverCapabilities.getBrowserVersion();
+ console.log( `\nDriver created for ${ name } ${ version }` );
+ }
+
+ // Increase script timeout to 10min
+ await driver.manage().setTimeouts( { script: DRIVER_SCRIPT_TIMEOUT } );
+
+ return driver;
+}
diff --git a/test/runner/selenium/runSelenium.js b/test/runner/selenium/runSelenium.js
new file mode 100644
index 000000000..247cd8472
--- /dev/null
+++ b/test/runner/selenium/runSelenium.js
@@ -0,0 +1,31 @@
+import chalk from "chalk";
+import createDriver from "./createDriver.js";
+
+export async function runSelenium(
+ url,
+ { browser },
+ { debug, headless, verbose } = {}
+) {
+ if ( debug && headless ) {
+ throw new Error( "Cannot debug in headless mode." );
+ }
+
+ const driver = await createDriver( {
+ browserName: browser,
+ headless,
+ verbose
+ } );
+
+ try {
+ await driver.get( url );
+ await driver.executeScript(
+`return new Promise( ( resolve ) => {
+ QUnit.on( "runEnd", resolve );
+} )`
+ );
+ } finally {
+ if ( !debug || headless ) {
+ await driver.quit();
+ }
+ }
+}
diff --git a/test/runner/server.js b/test/runner/server.js
new file mode 100644
index 000000000..09fe0da4c
--- /dev/null
+++ b/test/runner/server.js
@@ -0,0 +1,13 @@
+import { createTestServer } from "./createTestServer.js";
+
+const port = process.env.PORT || 3000;
+
+async function runServer() {
+ const app = await createTestServer();
+
+ app.listen( { port, host: "0.0.0.0" }, function() {
+ console.log( `Open tests at http://localhost:${ port }/test/` );
+ } );
+}
+
+runServer();