3 files changed, 381 insertions, 0 deletions
diff --git a/tests/runner/selenium/browsers.js b/tests/runner/selenium/browsers.js
new file mode 100644
index 000000000..568d6ed36
--- /dev/null
+++ b/tests/runner/selenium/browsers.js
@@ -0,0 +1,200 @@
+import chalk from "chalk";
+import { getBrowserString } from "../lib/getBrowserString.js";
+import createDriver from "./createDriver.js";
+
+const workers = Object.create( null );
+
+/**
+ * Keys are browser strings
+ * Structure of a worker:
+ * {
+ * 	 debug: boolean, // Stops the worker from being cleaned up when finished
+ *   id: string,
+ *   lastTouch: number, // The last time a request was received
+ *   url: string,
+ *   browser: object, // The browser object
+ *   options: object // The options to create the worker
+ * }
+ */
+
+// Acknowledge the worker within the time limit.
+const ACKNOWLEDGE_INTERVAL = 1000;
+const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 1;
+
+const MAX_WORKER_RESTARTS = 5;
+
+// No report after the time limit
+// should refresh the worker
+const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
+
+const WORKER_WAIT_TIME = 30000;
+
+// Limit concurrency to 8 by default in selenium
+const MAX_CONCURRENCY = 8;
+
+export function touchBrowser( browser ) {
+	const fullBrowser = getBrowserString( browser );
+	const worker = workers[ fullBrowser ];
+	if ( worker ) {
+		worker.lastTouch = Date.now();
+	}
+}
+
+async function waitForAck( worker, { fullBrowser, verbose } ) {
+	delete worker.lastTouch;
+	return new Promise( ( resolve, reject ) => {
+		const interval = setInterval( () => {
+			if ( worker.lastTouch ) {
+				if ( verbose ) {
+					console.log( `\n${ fullBrowser } acknowledged.` );
+				}
+				clearTimeout( timeout );
+				clearInterval( interval );
+				resolve();
+			}
+		}, ACKNOWLEDGE_INTERVAL );
+
+		const timeout = setTimeout( () => {
+			clearInterval( interval );
+			reject(
+				new Error(
+					`${ fullBrowser } not acknowledged after ${
+						ACKNOWLEDGE_TIMEOUT / 1000 / 60
+					}min.`
+				)
+			);
+		}, ACKNOWLEDGE_TIMEOUT );
+	} );
+}
+
+async function restartWorker( worker ) {
+	await cleanupWorker( worker, worker.options );
+	await createBrowserWorker(
+		worker.url,
+		worker.browser,
+		worker.options,
+		worker.restarts + 1
+	);
+}
+
+async function ensureAcknowledged( worker ) {
+	const fullBrowser = getBrowserString( worker.browser );
+	const verbose = worker.options.verbose;
+	try {
+		await waitForAck( worker, { fullBrowser, verbose } );
+		return worker;
+	} catch ( error ) {
+		console.error( error.message );
+		await restartWorker( worker );
+	}
+}
+
+export async function createBrowserWorker( url, browser, options, restarts = 0 ) {
+	if ( restarts > MAX_WORKER_RESTARTS ) {
+		throw new Error(
+			`Reached the maximum number of restarts for ${ chalk.yellow(
+				getBrowserString( browser )
+			) }`
+		);
+	}
+	const { concurrency = MAX_CONCURRENCY, debug, headless, verbose } = options;
+	while ( workers.length >= concurrency ) {
+		if ( verbose ) {
+			console.log( "\nWaiting for available sessions..." );
+		}
+		await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
+	}
+
+	const fullBrowser = getBrowserString( browser );
+
+	const driver = await createDriver( {
+		browserName: browser.browser,
+		headless,
+		url,
+		verbose
+	} );
+
+	const worker = {
+		debug: !!debug,
+		driver,
+		url,
+		browser,
+		restarts,
+		options
+	};
+
+	worker.debug = !!debug;
+	worker.url = url;
+	worker.browser = browser;
+	worker.restarts = restarts;
+	worker.options = options;
+	touchBrowser( browser );
+	workers[ fullBrowser ] = worker;
+
+	// Wait for the worker to show up in the list
+	// before returning it.
+	return ensureAcknowledged( worker );
+}
+
+export async function setBrowserWorkerUrl( browser, url ) {
+	const fullBrowser = getBrowserString( browser );
+	const worker = workers[ fullBrowser ];
+	if ( worker ) {
+		worker.url = url;
+	}
+}
+
+/**
+ * Checks that all browsers have received
+ * a response in the given amount of time.
+ * If not, the worker is restarted.
+ */
+export async function checkLastTouches() {
+	for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) {
+		if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) {
+			const options = worker.options;
+			if ( options.verbose ) {
+				console.log(
+					`\nNo response from ${ chalk.yellow( fullBrowser ) } in ${
+						RUN_WORKER_TIMEOUT / 1000 / 60
+					}min.`
+				);
+			}
+			await restartWorker( worker );
+		}
+	}
+}
+
+export async function cleanupWorker( worker, { verbose } ) {
+	for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
+		if ( w === worker ) {
+			delete workers[ fullBrowser ];
+			await w.driver.quit();
+			if ( verbose ) {
+				console.log( `\nStopped ${ fullBrowser }.` );
+			}
+			return;
+		}
+	}
+}
+
+export async function cleanupAllBrowsers( { verbose } ) {
+	const workersRemaining = Object.values( workers );
+	const numRemaining = workersRemaining.length;
+	if ( numRemaining ) {
+		try {
+			await Promise.all(
+				workersRemaining.map( ( worker ) => worker.driver.quit() )
+			);
+			if ( verbose ) {
+				console.log(
+					`Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
+				);
+			}
+		} catch ( error ) {
+
+			// Log the error, but do not consider the test run failed
+			console.error( error );
+		}
+	}
+}
diff --git a/tests/runner/selenium/createDriver.js b/tests/runner/selenium/createDriver.js
new file mode 100644
index 000000000..095c12214
--- /dev/null
+++ b/tests/runner/selenium/createDriver.js
@@ -0,0 +1,84 @@
+import { Builder, Capabilities, logging } from "selenium-webdriver";
+import Chrome from "selenium-webdriver/chrome.js";
+import Edge from "selenium-webdriver/edge.js";
+import Firefox from "selenium-webdriver/firefox.js";
+import { browserSupportsHeadless } from "../lib/getBrowserString.js";
+
+// Set script timeout to 10min
+const DRIVER_SCRIPT_TIMEOUT = 1000 * 60 * 10;
+
+export default async function createDriver( { browserName, headless, url, verbose } ) {
+	const capabilities = Capabilities[ browserName ]();
+	const prefs = new logging.Preferences();
+	prefs.setLevel( logging.Type.BROWSER, logging.Level.ALL );
+	capabilities.setLoggingPrefs( prefs );
+
+	let driver = new Builder().withCapabilities( capabilities );
+
+	const chromeOptions = new Chrome.Options();
+	chromeOptions.addArguments( "--enable-chrome-browser-cloud-management" );
+
+	// Alter the chrome binary path if
+	// the CHROME_BIN environment variable is set
+	if ( process.env.CHROME_BIN ) {
+		if ( verbose ) {
+			console.log( `Setting chrome binary to ${ process.env.CHROME_BIN }` );
+		}
+		chromeOptions.setChromeBinaryPath( process.env.CHROME_BIN );
+	}
+
+	const firefoxOptions = new Firefox.Options();
+
+	if ( process.env.FIREFOX_BIN ) {
+		if ( verbose ) {
+			console.log( `Setting firefox binary to ${ process.env.FIREFOX_BIN }` );
+		}
+
+		firefoxOptions.setBinary( process.env.FIREFOX_BIN );
+	}
+
+	const edgeOptions = new Edge.Options();
+	edgeOptions.addArguments( "--enable-chrome-browser-cloud-management" );
+
+	// Alter the edge binary path if
+	// the EDGE_BIN environment variable is set
+	if ( process.env.EDGE_BIN ) {
+		if ( verbose ) {
+			console.log( `Setting edge binary to ${ process.env.EDGE_BIN }` );
+		}
+		edgeOptions.setEdgeChromiumBinaryPath( process.env.EDGE_BIN );
+	}
+
+	if ( headless ) {
+		chromeOptions.addArguments( "--headless=new" );
+		firefoxOptions.addArguments( "--headless" );
+		edgeOptions.addArguments( "--headless=new" );
+		if ( !browserSupportsHeadless( browserName ) ) {
+			console.log(
+				`Headless mode is not supported for ${ browserName }.` +
+					"Running in normal mode instead."
+			);
+		}
+	}
+
+	driver = await driver
+		.setChromeOptions( chromeOptions )
+		.setFirefoxOptions( firefoxOptions )
+		.setEdgeOptions( edgeOptions )
+		.build();
+
+	if ( verbose ) {
+		const driverCapabilities = await driver.getCapabilities();
+		const name = driverCapabilities.getBrowserName();
+		const version = driverCapabilities.getBrowserVersion();
+		console.log( `\nDriver created for ${ name } ${ version }` );
+	}
+
+	// Increase script timeout to 10min
+	await driver.manage().setTimeouts( { script: DRIVER_SCRIPT_TIMEOUT } );
+
+	// Set the first URL for the browser
+	await driver.get( url );
+
+	return driver;
+}
diff --git a/tests/runner/selenium/queue.js b/tests/runner/selenium/queue.js
new file mode 100644
index 000000000..de24c5bb0
--- /dev/null
+++ b/tests/runner/selenium/queue.js
@@ -0,0 +1,97 @@
+import chalk from "chalk";
+import { getBrowserString } from "../lib/getBrowserString.js";
+import {
+	checkLastTouches,
+	createBrowserWorker,
+	setBrowserWorkerUrl
+} from "./browsers.js";
+
+const TEST_POLL_TIMEOUT = 1000;
+
+const queue = [];
+
+export function getNextBrowserTest( reportId ) {
+	const index = queue.findIndex( ( test ) => test.id === reportId );
+	if ( index === -1 ) {
+		return;
+	}
+
+	// Remove the completed test from the queue
+	const previousTest = queue[ index ];
+	queue.splice( index, 1 );
+
+	// Find the next test for the same browser
+	for ( const test of queue.slice( index ) ) {
+		if ( test.fullBrowser === previousTest.fullBrowser ) {
+
+			// Set the URL for our tracking
+			setBrowserWorkerUrl( test.browser, test.url );
+			test.running = true;
+
+			// Return the URL for the next test.
+			// listeners.js will use this to set the browser URL.
+			return { url: test.url };
+		}
+	}
+}
+
+export function retryTest( reportId, maxRetries ) {
+	if ( !maxRetries ) {
+		return;
+	}
+	const test = queue.find( ( test ) => test.id === reportId );
+	if ( test ) {
+		test.retries++;
+		if ( test.retries <= maxRetries ) {
+			console.log(
+				`\nRetrying test ${ reportId } for ${ chalk.yellow( test.options.suite ) }...${
+					test.retries
+				}`
+			);
+			return test;
+		}
+	}
+}
+
+export function addRun( url, browser, options ) {
+	queue.push( {
+		browser,
+		fullBrowser: getBrowserString( browser ),
+		id: options.reportId,
+		retries: 0,
+		url,
+		options,
+		running: false
+	} );
+}
+
+export async function runAll() {
+	return new Promise( async( resolve, reject ) => {
+		while ( queue.length ) {
+			try {
+				await checkLastTouches();
+			} catch ( error ) {
+				reject( error );
+			}
+
+			// Run one test URL per browser at a time
+			const browsersTaken = [];
+			for ( const test of queue ) {
+				if ( browsersTaken.indexOf( test.fullBrowser ) > -1 ) {
+					continue;
+				}
+				browsersTaken.push( test.fullBrowser );
+				if ( !test.running ) {
+					test.running = true;
+					try {
+						await createBrowserWorker( test.url, test.browser, test.options );
+					} catch ( error ) {
+						reject( error );
+					}
+				}
+			}
+			await new Promise( ( resolve ) => setTimeout( resolve, TEST_POLL_TIMEOUT ) );
+		}
+		resolve();
+	} );
+}