1 files changed, 234 insertions, 0 deletions
diff --git a/tests/runner/run.js b/tests/runner/run.js
new file mode 100644
index 000000000..bf3a16191
--- /dev/null
+++ b/tests/runner/run.js
@@ -0,0 +1,234 @@
+import chalk from "chalk";
+import { asyncExitHook, gracefulExit } from "exit-hook";
+import { reportEnd, reportTest } from "./reporter.js";
+import { createTestServer } from "./createTestServer.js";
+import { buildTestUrl } from "./lib/buildTestUrl.js";
+import { generateHash } from "./lib/generateHash.js";
+import { getBrowserString } from "./lib/getBrowserString.js";
+import { suites as allSuites } from "./suites.js";
+import { cleanupAllBrowsers, touchBrowser } from "./selenium/browsers.js";
+import { addRun, getNextBrowserTest, retryTest, runAll } from "./selenium/queue.js";
+
+const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
+
+/**
+ * Run test suites in parallel in different browser instances.
+ */
+export async function run( {
+	browser: browserNames = [],
+	concurrency,
+	debug,
+	headless,
+	jquery: jquerys = [],
+	migrate,
+	retries = 0,
+	suite: suites = [],
+	verbose
+} ) {
+	if ( !browserNames.length ) {
+		browserNames = [ "chrome" ];
+	}
+	if ( !suites.length ) {
+		suites = allSuites;
+	}
+	if ( !jquerys.length ) {
+		jquerys = [ "3.7.1" ];
+	}
+	if ( headless && debug ) {
+		throw new Error(
+			"Cannot run in headless mode and debug mode at the same time."
+		);
+	}
+
+	const errorMessages = [];
+	const pendingErrors = {};
+
+	// Convert browser names to browser objects
+	let browsers = browserNames.map( ( b ) => ( { browser: b } ) );
+
+	// Create the test app and
+	// hook it up to the reporter
+	const reports = Object.create( null );
+	const app = await createTestServer( async( message ) => {
+		switch ( message.type ) {
+			case "testEnd": {
+				const reportId = message.id;
+				const report = reports[ reportId ];
+				touchBrowser( report.browser );
+				const errors = reportTest( message.data, reportId, report );
+				pendingErrors[ reportId ] ??= Object.create( null );
+				if ( errors ) {
+					pendingErrors[ reportId ][ message.data.name ] = errors;
+				} else {
+					const existing = pendingErrors[ reportId ][ message.data.name ];
+
+					// Show a message for flakey tests
+					if ( existing ) {
+						console.log();
+						console.warn(
+							chalk.italic(
+								chalk.gray( existing.replace( "Test failed", "Test flakey" ) )
+							)
+						);
+						console.log();
+						delete pendingErrors[ reportId ][ message.data.name ];
+					}
+				}
+				break;
+			}
+			case "runEnd": {
+				const reportId = message.id;
+				const report = reports[ reportId ];
+				touchBrowser( report.browser );
+				const { failed, total } = reportEnd(
+					message.data,
+					message.id,
+					reports[ reportId ]
+				);
+				report.total = total;
+
+				// Handle failure
+				if ( failed ) {
+					const retry = retryTest( reportId, retries );
+
+					// Retry if retryTest returns a test
+					if ( retry ) {
+						return retry;
+					}
+
+					errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
+				}
+
+				// Run the next test
+				return getNextBrowserTest( reportId );
+			}
+			case "ack": {
+				const report = reports[ message.id ];
+				touchBrowser( report.browser );
+				break;
+			}
+			default:
+				console.warn( "Received unknown message type:", message.type );
+		}
+	} );
+
+	// Start up local test server
+	let server;
+	let port;
+	await new Promise( ( resolve ) => {
+
+		// Pass 0 to choose a random, unused port
+		server = app.listen( 0, () => {
+			port = server.address().port;
+			resolve();
+		} );
+	} );
+
+	if ( !server || !port ) {
+		throw new Error( "Server not started." );
+	}
+
+	if ( verbose ) {
+		console.log( `Server started on port ${ port }.` );
+	}
+
+	function stopServer() {
+		return new Promise( ( resolve ) => {
+			server.close( () => {
+				if ( verbose ) {
+					console.log( "Server stopped." );
+				}
+				resolve();
+			} );
+		} );
+	}
+
+	asyncExitHook(
+		async() => {
+			await cleanupAllBrowsers( { verbose } );
+			await stopServer();
+		},
+		{ wait: EXIT_HOOK_WAIT_TIMEOUT }
+	);
+
+	function queueRuns( suite, browser ) {
+		const fullBrowser = getBrowserString( browser, headless );
+
+		for ( const jquery of jquerys ) {
+			const reportId = generateHash( `${ suite } ${ fullBrowser }` );
+			reports[ reportId ] = { browser, headless, jquery, migrate, suite };
+
+			const url = buildTestUrl( suite, {
+				jquery,
+				migrate,
+				port,
+				reportId
+			} );
+
+			const options = {
+				debug,
+				headless,
+				jquery,
+				migrate,
+				reportId,
+				suite,
+				verbose
+			};
+
+			addRun( url, browser, options );
+		}
+	}
+
+	for ( const browser of browsers ) {
+		for ( const suite of suites ) {
+			queueRuns( suite, browser );
+		}
+	}
+
+	try {
+		await runAll( { concurrency, verbose } );
+	} catch ( error ) {
+		console.error( error );
+		if ( !debug ) {
+			gracefulExit( 1 );
+		}
+	} finally {
+		console.log();
+		if ( errorMessages.length === 0 ) {
+			let stop = false;
+			for ( const report of Object.values( reports ) ) {
+				if ( !report.total ) {
+					stop = true;
+					console.error(
+						chalk.red(
+							`No tests were run for ${ report.suite } in ${ getBrowserString(
+								report.browser
+							) }`
+						)
+					);
+				}
+			}
+			if ( stop ) {
+				return gracefulExit( 1 );
+			}
+			console.log( chalk.green( "All tests passed!" ) );
+
+			if ( !debug ) {
+				gracefulExit( 0 );
+			}
+		} else {
+			console.error( chalk.red( `${ errorMessages.length } tests failed.` ) );
+			console.log(
+				errorMessages.map( ( error, i ) => `\n${ i + 1 }. ${ error }` ).join( "\n" )
+			);
+
+			if ( debug ) {
+				console.log();
+				console.log( "Leaving browsers open for debugging." );
+				console.log( "Press Ctrl+C to exit." );
+			} else {
+				gracefulExit( 1 );
+			}
+		}
+	}
+}