diff options
Diffstat (limited to 'test/runner')
-rw-r--r-- | test/runner/browsers.js | 25 | ||||
-rw-r--r-- | test/runner/browserstack/api.js | 289 | ||||
-rw-r--r-- | test/runner/browserstack/buildBrowserFromString.js | 10 | ||||
-rw-r--r-- | test/runner/browserstack/createAuthHeader.js | 7 | ||||
-rw-r--r-- | test/runner/browserstack/local.js | 34 | ||||
-rw-r--r-- | test/runner/browserstack/workers.js | 276 | ||||
-rw-r--r-- | test/runner/command.js | 120 | ||||
-rw-r--r-- | test/runner/createTestServer.js | 60 | ||||
-rw-r--r-- | test/runner/jsdom.js | 55 | ||||
-rw-r--r-- | test/runner/lib/buildTestUrl.js | 29 | ||||
-rw-r--r-- | test/runner/lib/generateHash.js | 50 | ||||
-rw-r--r-- | test/runner/lib/getBrowserString.js | 49 | ||||
-rw-r--r-- | test/runner/lib/prettyMs.js | 18 | ||||
-rw-r--r-- | test/runner/listeners.js | 88 | ||||
-rw-r--r-- | test/runner/modules.js | 24 | ||||
-rw-r--r-- | test/runner/queue.js | 102 | ||||
-rw-r--r-- | test/runner/reporter.js | 54 | ||||
-rw-r--r-- | test/runner/run.js | 315 | ||||
-rw-r--r-- | test/runner/selenium/createDriver.js | 80 | ||||
-rw-r--r-- | test/runner/selenium/runSelenium.js | 31 | ||||
-rw-r--r-- | test/runner/server.js | 13 |
21 files changed, 1729 insertions, 0 deletions
diff --git a/test/runner/browsers.js b/test/runner/browsers.js new file mode 100644 index 000000000..a3a8df0d8 --- /dev/null +++ b/test/runner/browsers.js @@ -0,0 +1,25 @@ +// This list is static, so no requests are required +// in the command help menu. + +import { getBrowsers } from "./browserstack/api.js"; + +export const browsers = [ + "chrome", + "ie", + "firefox", + "edge", + "safari", + "opera", + "yandex", + "IE Mobile", + "Android Browser", + "Mobile Safari", + "jsdom" +]; + +// A function that can be used to update the above list. +export async function getAvailableBrowsers() { + const browsers = await getBrowsers( { flat: true } ); + const available = [ ...new Set( browsers.map( ( { browser } ) => browser ) ) ]; + return available.concat( "jsdom" ); +} diff --git a/test/runner/browserstack/api.js b/test/runner/browserstack/api.js new file mode 100644 index 000000000..40982c9b3 --- /dev/null +++ b/test/runner/browserstack/api.js @@ -0,0 +1,289 @@ +/** + * Browserstack API is documented at + * https://github.com/browserstack/api + */ + +import { createAuthHeader } from "./createAuthHeader.js"; + +const browserstackApi = "https://api.browserstack.com"; +const apiVersion = 5; + +const username = process.env.BROWSERSTACK_USERNAME; +const accessKey = process.env.BROWSERSTACK_ACCESS_KEY; + +// iOS has null for version numbers, +// and we do not need a similar check for OS versions. +const rfinalVersion = /(?:^[0-9\.]+$)|(?:^null$)/; + +const rnonDigits = /(?:[^\d\.]+)|(?:20\d{2})/g; + +async function fetchAPI( path, options = {}, versioned = true ) { + if ( !username || !accessKey ) { + throw new Error( + "BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables must be set." + ); + } + const init = { + method: "GET", + ...options, + headers: { + authorization: createAuthHeader( username, accessKey ), + accept: "application/json", + "content-type": "application/json", + ...options.headers + } + }; + const response = await fetch( + `${ browserstackApi }/${ versioned ? `${ apiVersion }/` : "" }${ path }`, + init + ); + if ( !response.ok ) { + console.log( + `\n${ init.method } ${ path }`, + response.status, + response.statusText + ); + throw new Error( `Error fetching ${ path }` ); + } + return response.json(); +} + +/** + * ============================= + * Browsers API + * ============================= + */ + +function compareVersionNumbers( a, b ) { + if ( a != null && b == null ) { + return -1; + } + if ( a == null && b != null ) { + return 1; + } + if ( a == null && b == null ) { + return 0; + } + const aParts = a.replace( rnonDigits, "" ).split( "." ); + const bParts = b.replace( rnonDigits, "" ).split( "." ); + + if ( aParts.length > bParts.length ) { + return -1; + } + if ( aParts.length < bParts.length ) { + return 1; + } + + for ( let i = 0; i < aParts.length; i++ ) { + const aPart = Number( aParts[ i ] ); + const bPart = Number( bParts[ i ] ); + if ( aPart < bPart ) { + return -1; + } + if ( aPart > bPart ) { + return 1; + } + } +} + +function sortBrowsers( a, b ) { + if ( a.browser < b.browser ) { + return -1; + } + if ( a.browser > b.browser ) { + return 1; + } + const browserComparison = compareVersionNumbers( + a.browser_version, + b.browser_version + ); + if ( browserComparison ) { + return browserComparison; + } + if ( a.os < b.os ) { + return -1; + } + if ( a.os > b.os ) { + return 1; + } + const osComparison = compareVersionNumbers( a.os_version, b.os_version ); + if ( osComparison ) { + return osComparison; + } + const deviceComparison = compareVersionNumbers( a.device, b.device ); + if ( deviceComparison ) { + return deviceComparison; + } + return 0; +} + +export async function getBrowsers( { flat = false } = {} ) { + const query = new URLSearchParams(); + if ( flat ) { + query.append( "flat", true ); + } + const browsers = await fetchAPI( `/browsers?${ query }` ); + return browsers.sort( sortBrowsers ); +} + +function matchVersion( browserVersion, version ) { + if ( !version ) { + return false; + } + const regex = new RegExp( + `^${ version.replace( /\\/g, "\\\\" ).replace( /\./g, "\\." ) }\\b`, + "i" + ); + return regex.test( browserVersion ); +} + +export async function filterBrowsers( filter ) { + const browsers = await getBrowsers( { flat: true } ); + if ( !filter ) { + return browsers; + } + const filterBrowser = ( filter.browser ?? "" ).toLowerCase(); + const filterVersion = ( filter.browser_version ?? "" ).toLowerCase(); + const filterOs = ( filter.os ?? "" ).toLowerCase(); + const filterOsVersion = ( filter.os_version ?? "" ).toLowerCase(); + const filterDevice = ( filter.device ?? "" ).toLowerCase(); + + return browsers.filter( ( browser ) => { + return ( + ( !filterBrowser || filterBrowser === browser.browser.toLowerCase() ) && + ( !filterVersion || + matchVersion( browser.browser_version, filterVersion ) ) && + ( !filterOs || filterOs === browser.os.toLowerCase() ) && + ( !filterOsVersion || + filterOsVersion === browser.os_version.toLowerCase() ) && + ( !filterDevice || filterDevice === ( browser.device || "" ).toLowerCase() ) + ); + } ); +} + +export async function listBrowsers( filter ) { + const browsers = await filterBrowsers( filter ); + console.log( "Available browsers:" ); + for ( const browser of browsers ) { + let message = ` ${ browser.browser }_`; + if ( browser.device ) { + message += `:${ browser.device }_`; + } else { + message += `${ browser.browser_version }_`; + } + message += `${ browser.os }_${ browser.os_version }`; + console.log( message ); + } +} + +export async function getLatestBrowser( filter ) { + const browsers = await filterBrowsers( filter ); + + // The list is sorted in ascending order, + // so the last item is the latest. + return browsers.findLast( ( browser ) => + rfinalVersion.test( browser.browser_version ) + ); +} + +/** + * ============================= + * Workers API + * ============================= + */ + +/** + * A browser object may only have one of `browser` or `device` set; + * which property is set will depend on `os`. + * + * `options`: is an object with the following properties: + * `os`: The operating system. + * `os_version`: The operating system version. + * `browser`: The browser name. + * `browser_version`: The browser version. + * `device`: The device name. + * `url` (optional): Which URL to navigate to upon creation. + * `timeout` (optional): Maximum life of the worker (in seconds). Maximum value of `1800`. Specifying `0` will use the default of `300`. + * `name` (optional): Provide a name for the worker. + * `build` (optional): Group workers into a build. + * `project` (optional): Provide the project the worker belongs to. + * `resolution` (optional): Specify the screen resolution (e.g. "1024x768"). + * `browserstack.local` (optional): Set to `true` to mark as local testing. + * `browserstack.video` (optional): Set to `false` to disable video recording. + * `browserstack.localIdentifier` (optional): ID of the local tunnel. + */ +export function createWorker( options ) { + return fetchAPI( "/worker", { + method: "POST", + body: JSON.stringify( options ) + } ); +} + +/** + * Returns a worker object, if one exists, with the following properties: + * `id`: The worker id. + * `status`: A string representing the current status of the worker. + * Possible statuses: `"running"`, `"queue"`. + */ +export function getWorker( id ) { + return fetchAPI( `/worker/${ id }` ); +} + +export async function deleteWorker( id, verbose ) { + await fetchAPI( `/worker/${ id }`, { method: "DELETE" } ); + if ( verbose ) { + console.log( `\nWorker ${ id } stopped.` ); + } +} + +export function getWorkers() { + return fetchAPI( "/workers" ); +} + +/** + * Change the URL of a worker, + * or refresh if it's the same URL. + */ +export function changeUrl( id, url ) { + return fetchAPI( `/worker/${ id }/url.json`, { + method: "PUT", + body: JSON.stringify( { + timeout: 20, + url: encodeURI( url ) + } ) + } ); +} + +/** + * Stop all workers + */ +export async function stopWorkers() { + const workers = await getWorkers(); + + // Run each request on its own + // to avoid connect timeout errors. + for ( const worker of workers ) { + try { + await deleteWorker( worker.id, true ); + } catch ( error ) { + + // Log the error, but continue trying to remove workers. + console.error( error ); + } + } +} + +/** + * ============================= + * Plan API + * ============================= + */ + +export function getPlan() { + return fetchAPI( "/automate/plan.json", {}, false ); +} + +export async function getAvailableSessions() { + const [ plan, workers ] = await Promise.all( [ getPlan(), getWorkers() ] ); + return plan.parallel_sessions_max_allowed - workers.length; +} diff --git a/test/runner/browserstack/buildBrowserFromString.js b/test/runner/browserstack/buildBrowserFromString.js new file mode 100644 index 000000000..55aa38053 --- /dev/null +++ b/test/runner/browserstack/buildBrowserFromString.js @@ -0,0 +1,10 @@ +export function buildBrowserFromString( str ) { + const [ browser, versionOrDevice, os, osVersion ] = str.split( "_" ); + + // If the version starts with a colon, it's a device + if ( versionOrDevice && versionOrDevice.startsWith( ":" ) ) { + return { browser, device: versionOrDevice.slice( 1 ), os, os_version: osVersion }; + } + + return { browser, browser_version: versionOrDevice, os, os_version: osVersion }; +} diff --git a/test/runner/browserstack/createAuthHeader.js b/test/runner/browserstack/createAuthHeader.js new file mode 100644 index 000000000..fe4831e9a --- /dev/null +++ b/test/runner/browserstack/createAuthHeader.js @@ -0,0 +1,7 @@ +const textEncoder = new TextEncoder(); + +export function createAuthHeader( username, accessKey ) { + const encoded = textEncoder.encode( `${ username }:${ accessKey }` ); + const base64 = btoa( String.fromCodePoint.apply( null, encoded ) ); + return `Basic ${ base64 }`; +} diff --git a/test/runner/browserstack/local.js b/test/runner/browserstack/local.js new file mode 100644 index 000000000..c84cf155c --- /dev/null +++ b/test/runner/browserstack/local.js @@ -0,0 +1,34 @@ +import browserstackLocal from "browserstack-local"; + +export async function localTunnel( localIdentifier, opts = {} ) { + const tunnel = new browserstackLocal.Local(); + + return new Promise( ( resolve, reject ) => { + + // https://www.browserstack.com/docs/local-testing/binary-params + tunnel.start( + { + "enable-logging-for-api": "", + localIdentifier, + ...opts + }, + async( error ) => { + if ( error || !tunnel.isRunning() ) { + return reject( error ); + } + resolve( { + stop: function stopTunnel() { + return new Promise( ( resolve, reject ) => { + tunnel.stop( ( error ) => { + if ( error ) { + return reject( error ); + } + resolve(); + } ); + } ); + } + } ); + } + ); + } ); +} diff --git a/test/runner/browserstack/workers.js b/test/runner/browserstack/workers.js new file mode 100644 index 000000000..8f0ab68f0 --- /dev/null +++ b/test/runner/browserstack/workers.js @@ -0,0 +1,276 @@ +import chalk from "chalk"; +import { getBrowserString } from "../lib/getBrowserString.js"; +import { changeUrl, createWorker, deleteWorker, getWorker } from "./api.js"; + +const workers = Object.create( null ); + +// Acknowledge the worker within the time limit. +// BrowserStack can take much longer spinning up +// some browsers, such as iOS 15 Safari. +const ACKNOWLEDGE_WORKER_TIMEOUT = 60 * 1000 * 8; +const ACKNOWLEDGE_WORKER_INTERVAL = 1000; + +// No report after the time limit +// should refresh the worker +const RUN_WORKER_TIMEOUT = 60 * 1000 * 2; +const MAX_WORKER_RESTARTS = 5; +const MAX_WORKER_REFRESHES = 1; +const POLL_WORKER_TIMEOUT = 1000; + +export async function cleanupWorker( reportId, verbose ) { + const worker = workers[ reportId ]; + if ( worker ) { + try { + delete workers[ reportId ]; + await deleteWorker( worker.id, verbose ); + } catch ( error ) { + console.error( error ); + } + } +} + +export function debugWorker( reportId ) { + const worker = workers[ reportId ]; + if ( worker ) { + worker.debug = true; + } +} + +/** + * Set the last time a request was + * received related to the worker. + */ +export function touchWorker( reportId ) { + const worker = workers[ reportId ]; + if ( worker ) { + worker.lastTouch = Date.now(); + } +} + +export function retryTest( reportId, retries ) { + const worker = workers[ reportId ]; + if ( worker ) { + worker.retries ||= 0; + worker.retries++; + if ( worker.retries <= retries ) { + worker.retry = true; + console.log( `\nRetrying test ${ reportId }...${ worker.retries }` ); + return true; + } + } + return false; +} + +export async function cleanupAllWorkers( verbose ) { + const workersRemaining = Object.keys( workers ).length; + if ( workersRemaining ) { + if ( verbose ) { + console.log( + `Stopping ${ workersRemaining } stray worker${ + workersRemaining > 1 ? "s" : "" + }...` + ); + } + await Promise.all( + Object.values( workers ).map( ( worker ) => deleteWorker( worker.id, verbose ) ) + ); + } +} + +async function waitForAck( id, verbose ) { + return new Promise( ( resolve, reject ) => { + const interval = setInterval( () => { + const worker = workers[ id ]; + if ( !worker ) { + clearTimeout( timeout ); + clearInterval( interval ); + return reject( new Error( `Worker ${ id } not found.` ) ); + } + if ( worker.lastTouch ) { + if ( verbose ) { + console.log( `\nWorker ${ id } acknowledged.` ); + } + clearTimeout( timeout ); + clearInterval( interval ); + resolve(); + } + }, ACKNOWLEDGE_WORKER_INTERVAL ); + const timeout = setTimeout( () => { + clearInterval( interval ); + const worker = workers[ id ]; + reject( + new Error( + `Worker ${ + worker ? worker.id : "" + } for test ${ id } not acknowledged after ${ + ACKNOWLEDGE_WORKER_TIMEOUT / 1000 + }s.` + ) + ); + }, ACKNOWLEDGE_WORKER_TIMEOUT ); + } ); +} + +export async function runWorker( + url, + browser, + options, + restarts = 0 +) { + const { modules, reportId, runId, verbose } = options; + const worker = await createWorker( { + ...browser, + url: encodeURI( url ), + project: "jquery", + build: `Run ${ runId }`, + name: `${ modules.join( "," ) } (${ reportId })`, + + // Set the max here, so that we can + // control the timeout + timeout: 1800, + + // Not documented in the API docs, + // but required to make local testing work. + // See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs + "browserstack.local": true, + "browserstack.localIdentifier": runId + } ); + + workers[ reportId ] = worker; + + const timeMessage = `\nWorker ${ + worker.id + } created for test ${ reportId } (${ chalk.yellow( getBrowserString( browser ) ) })`; + + if ( verbose ) { + console.time( timeMessage ); + } + + async function retryWorker() { + await cleanupWorker( reportId, verbose ); + if ( verbose ) { + console.log( `Retrying worker for test ${ reportId }...${ restarts + 1 }` ); + } + return runWorker( url, browser, options, restarts + 1 ); + } + + // Wait for the worker to be acknowledged + try { + await waitForAck( reportId ); + } catch ( error ) { + if ( !workers[ reportId ] ) { + + // The worker has already been cleaned up + return; + } + + if ( restarts < MAX_WORKER_RESTARTS ) { + return retryWorker(); + } + + throw error; + } + + if ( verbose ) { + console.timeEnd( timeMessage ); + } + + let refreshes = 0; + let loggedStarted = false; + return new Promise( ( resolve, reject ) => { + async function refreshWorker() { + try { + await changeUrl( worker.id, url ); + touchWorker( reportId ); + return tick(); + } catch ( error ) { + if ( !workers[ reportId ] ) { + + // The worker has already been cleaned up + return resolve(); + } + console.error( error ); + return retryWorker().then( resolve, reject ); + } + } + + async function checkWorker() { + const worker = workers[ reportId ]; + + if ( !worker || worker.debug ) { + return resolve(); + } + + let fetchedWorker; + try { + fetchedWorker = await getWorker( worker.id ); + } catch ( error ) { + return reject( error ); + } + if ( + !fetchedWorker || + ( fetchedWorker.status !== "running" && fetchedWorker.status !== "queue" ) + ) { + return resolve(); + } + + if ( verbose && !loggedStarted ) { + loggedStarted = true; + console.log( + `\nTest ${ chalk.bold( reportId ) } is ${ + worker.status === "running" ? "running" : "in the queue" + }.` + ); + console.log( ` View at ${ fetchedWorker.browser_url }.` ); + } + + // Refresh the worker if a retry is triggered + if ( worker.retry ) { + worker.retry = false; + + // Reset recovery refreshes + refreshes = 0; + return refreshWorker(); + } + + if ( worker.lastTouch > Date.now() - RUN_WORKER_TIMEOUT ) { + return tick(); + } + + refreshes++; + + if ( refreshes >= MAX_WORKER_REFRESHES ) { + if ( restarts < MAX_WORKER_RESTARTS ) { + if ( verbose ) { + console.log( + `Worker ${ worker.id } not acknowledged after ${ + ACKNOWLEDGE_WORKER_TIMEOUT / 1000 + }s.` + ); + } + return retryWorker().then( resolve, reject ); + } + await cleanupWorker( reportId, verbose ); + return reject( + new Error( + `Worker ${ worker.id } for test ${ reportId } timed out after ${ MAX_WORKER_RESTARTS } restarts.` + ) + ); + } + + if ( verbose ) { + console.log( + `\nRefreshing worker ${ worker.id } for test ${ reportId }...${ refreshes }` + ); + } + + return refreshWorker(); + } + + function tick() { + setTimeout( checkWorker, POLL_WORKER_TIMEOUT ); + } + + checkWorker(); + } ); +} diff --git a/test/runner/command.js b/test/runner/command.js new file mode 100644 index 000000000..8419625a7 --- /dev/null +++ b/test/runner/command.js @@ -0,0 +1,120 @@ +import yargs from "yargs/yargs"; +import { browsers } from "./browsers.js"; +import { getPlan, listBrowsers, stopWorkers } from "./browserstack/api.js"; +import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js"; +import { modules } from "./modules.js"; +import { run } from "./run.js"; + +const argv = yargs( process.argv.slice( 2 ) ) + .version( false ) + .command( { + command: "[options]", + describe: "Run jQuery tests in a browser" + } ) + .option( "module", { + alias: "m", + type: "array", + choices: modules, + description: + "Run tests for a specific module. " + + "Pass multiple modules by repeating the option. " + + "Defaults to all modules." + } ) + .option( "browser", { + alias: "b", + type: "array", + choices: browsers, + description: + "Run tests in a specific browser." + + "Pass multiple browsers by repeating the option." + + "If using BrowserStack, specify browsers using --browserstack." + + "Only the basic module is supported on jsdom.", + default: [ "chrome" ] + } ) + .option( "headless", { + alias: "h", + type: "boolean", + description: + "Run tests in headless mode. Cannot be used with --debug or --browserstack.", + conflicts: [ "debug", "browserstack" ] + } ) + .option( "esm", { + alias: "esmodules", + type: "boolean", + description: "Run tests using jQuery's source, which is written with ECMAScript Modules." + } ) + .option( "concurrency", { + alias: "c", + type: "number", + description: + "Run tests in parallel in multiple browsers. " + + "Defaults to 8 in normal mode. In browserstack mode, defaults to the maximum available under your BrowserStack plan." + } ) + .option( "debug", { + alias: "d", + type: "boolean", + description: + "Leave the browser open for debugging. Cannot be used with --headless.", + conflicts: [ "headless" ] + } ) + .option( "verbose", { + alias: "v", + type: "boolean", + description: "Log additional information." + } ) + .option( "retries", { + alias: "r", + type: "number", + description: "Number of times to retry failed tests.", + default: 0 + } ) + .option( "no-isolate", { + type: "boolean", + description: "Run all modules in the same browser instance." + } ) + .option( "browserstack", { + type: "array", + description: + "Run tests in BrowserStack.\nRequires BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables.\n" + + "The value can be empty for the default configuration, or a string in the format of\n" + + "\"browser_[browserVersion | :device]_os_osVersion\" (see --list-browsers).\n" + + "Pass multiple browsers by repeating the option. The --browser option is ignored when --browserstack has a value.\n" + + "Otherwise, the --browser option will be used, with the latest version/device for that browser, on a matching OS." + } ) + .option( "list-browsers", { + type: "string", + description: + "List available BrowserStack browsers and exit.\n" + + "Leave blank to view all browsers or pass " + + "\"browser_[browserVersion | :device]_os_osVersion\" with each parameter " + + "separated by an underscore to filter the list (any can be omitted).\n" + + "Use a colon to indicate a device.\n" + + "Examples: \"chrome__windows_10\", \"Mobile Safari\", \"Android Browser_:Google Pixel 8 Pro\".\n" + + "Use quotes if spaces are necessary." + } ) + .option( "stop-workers", { + type: "boolean", + description: + "WARNING: This will stop all BrowserStack workers that may exist and exit," + + "including any workers running from other projects.\n" + + "This can be used as a failsafe when there are too many stray workers." + } ) + .option( "browserstack-plan", { + type: "boolean", + description: "Show BrowserStack plan information and exit." + } ) + .help().argv; + +if ( typeof argv.listBrowsers === "string" ) { + listBrowsers( buildBrowserFromString( argv.listBrowsers ) ); +} else if ( argv.stopWorkers ) { + stopWorkers(); +} else if ( argv.browserstackPlan ) { + console.log( await getPlan() ); +} else { + run( { + ...argv, + browsers: argv.browser, + modules: argv.module + } ); +} diff --git a/test/runner/createTestServer.js b/test/runner/createTestServer.js new file mode 100644 index 000000000..ebc6bd4bb --- /dev/null +++ b/test/runner/createTestServer.js @@ -0,0 +1,60 @@ +import bodyParser from "body-parser"; +import express from "express"; +import bodyParserErrorHandler from "express-body-parser-error-handler"; +import fs from "fs"; +import mockServer from "../middleware-mockserver.cjs"; + +export async function createTestServer( report ) { + const nameHTML = await fs.promises.readFile( "./test/data/name.html", "utf8" ); + const indexHTML = await fs.promises.readFile( "./test/index.html", "utf8" ); + const app = express(); + + // Redirect home to test page + app.get( "/", ( _req, res ) => { + res.redirect( "/test/" ); + } ); + + // Redirect to trailing slash + app.use( ( req, res, next ) => { + if ( req.path === "/test" ) { + const query = req.url.slice( req.path.length ); + res.redirect( 301, `${ req.path }/${ query }` ); + } else { + next(); + } + } ); + + // Add a script tag to the index.html to load the QUnit listeners + app.use( /\/test(?:\/index.html)?\//, ( _req, res ) => { + res.send( indexHTML.replace( + "</head>", + "<script src=\"/test/runner/listeners.js\"></script></head>" + ) ); + } ); + + // Bind the reporter + app.post( "/api/report", bodyParser.json( { limit: "50mb" } ), ( req, res ) => { + if ( report ) { + report( req.body ); + } + res.sendStatus( 204 ); + } ); + + // Handle errors from the body parser + app.use( bodyParserErrorHandler() ); + + // Hook up mock server + app.use( mockServer() ); + + // Serve static files + app.post( "/test/data/name.html", ( _req, res ) => { + res.send( nameHTML ); + } ); + + app.use( "/dist", express.static( "dist" ) ); + app.use( "/src", express.static( "src" ) ); + app.use( "/test", express.static( "test" ) ); + app.use( "/external", express.static( "external" ) ); + + return app; +} diff --git a/test/runner/jsdom.js b/test/runner/jsdom.js new file mode 100644 index 000000000..d370ac348 --- /dev/null +++ b/test/runner/jsdom.js @@ -0,0 +1,55 @@ +import jsdom from "jsdom"; + +const { JSDOM } = jsdom; + +const windows = Object.create( null ); + +export async function runJSDOM( url, { reportId, verbose } ) { + const virtualConsole = new jsdom.VirtualConsole(); + virtualConsole.sendTo( console ); + virtualConsole.removeAllListeners( "clear" ); + + const { window } = await JSDOM.fromURL( url, { + resources: "usable", + runScripts: "dangerously", + virtualConsole + } ); + if ( verbose ) { + console.log( "JSDOM window opened.", reportId ); + } + windows[ reportId ] = window; + + return new Promise( ( resolve ) => { + window.finish = resolve; + } ); +} + +export function cleanupJSDOM( reportId, verbose ) { + const window = windows[ reportId ]; + if ( window ) { + if ( window.finish ) { + window.finish(); + } + window.close(); + delete windows[ reportId ]; + if ( verbose ) { + console.log( "Closed JSDOM window.", reportId ); + } + } +} + +export function cleanupAllJSDOM( verbose ) { + const windowsRemaining = Object.keys( windows ).length; + if ( windowsRemaining ) { + if ( verbose ) { + console.log( + `Cleaning up ${ windowsRemaining } JSDOM window${ + windowsRemaining > 1 ? "s" : "" + }...` + ); + } + for ( const id in windows ) { + cleanupJSDOM( id, verbose ); + } + } +} diff --git a/test/runner/lib/buildTestUrl.js b/test/runner/lib/buildTestUrl.js new file mode 100644 index 000000000..6e0f1a9b0 --- /dev/null +++ b/test/runner/lib/buildTestUrl.js @@ -0,0 +1,29 @@ +import { generateModuleId } from "./generateHash.js"; + +export function buildTestUrl( modules, { browserstack, esm, jsdom, port, reportId } ) { + if ( !port ) { + throw new Error( "No port specified." ); + } + + const query = new URLSearchParams(); + for ( const module of modules ) { + query.append( "moduleId", generateModuleId( module ) ); + } + + if ( esm ) { + query.append( "esmodules", "true" ); + } + + if ( jsdom ) { + query.append( "jsdom", "true" ); + } + + if ( reportId ) { + query.append( "reportId", reportId ); + } + + // BrowserStack supplies a custom domain for local testing, + // which is especially necessary for iOS testing. + const host = browserstack ? "bs-local.com" : "localhost"; + return `http://${ host }:${ port }/test/?${ query }`; +} diff --git a/test/runner/lib/generateHash.js b/test/runner/lib/generateHash.js new file mode 100644 index 000000000..dbbd4b476 --- /dev/null +++ b/test/runner/lib/generateHash.js @@ -0,0 +1,50 @@ +import crypto from "node:crypto"; + +export function generateHash( string ) { + const hash = crypto.createHash( "md5" ); + hash.update( string ); + + // QUnit hashes are 8 characters long + // We use 10 characters to be more visually distinct + return hash.digest( "hex" ).slice( 0, 10 ); +} + +/** + * A copy of the generate hash function from QUnit, + * used to generate a hash for the module name. + * + * QUnit errors on passing multiple modules to the + * module query parameter. We need to know + * the hash for each module before loading QUnit + * in order to pass multiple moduleId parameters instead. + */ +export function generateModuleId( module, browser ) { + + // QUnit normally hashes the test name, but + // we've repurposed this function to generate + // report IDs for module/browser combinations. + // We still use it without the browser parameter + // to get the same module IDs as QUnit to pass + // multiple ahead-of-time in the query string. + const str = module + "\x1C" + browser; + let hash = 0; + + for ( let i = 0; i < str.length; i++ ) { + hash = ( hash << 5 ) - hash + str.charCodeAt( i ); + hash |= 0; + } + + let hex = ( 0x100000000 + hash ).toString( 16 ); + if ( hex.length < 8 ) { + hex = "0000000" + hex; + } + + return hex.slice( -8 ); +} + +export function printModuleHashes( modules ) { + console.log( "Module hashes:" ); + modules.forEach( ( module ) => { + console.log( ` ${ module }: ${ generateModuleId( module ) }` ); + } ); +} diff --git a/test/runner/lib/getBrowserString.js b/test/runner/lib/getBrowserString.js new file mode 100644 index 000000000..413a60500 --- /dev/null +++ b/test/runner/lib/getBrowserString.js @@ -0,0 +1,49 @@ +const browserMap = { + chrome: "Chrome", + edge: "Edge", + firefox: "Firefox", + ie: "IE", + jsdom: "JSDOM", + opera: "Opera", + safari: "Safari" +}; + +export function browserSupportsHeadless( browser ) { + browser = browser.toLowerCase(); + return ( + browser === "chrome" || + browser === "firefox" || + browser === "edge" + ); +} + +export function getBrowserString( + { + browser, + browser_version: browserVersion, + device, + os, + os_version: osVersion + }, + headless +) { + browser = browser.toLowerCase(); + browser = browserMap[ browser ] || browser; + let str = browser; + if ( browserVersion ) { + str += ` ${ browserVersion }`; + } + if ( device ) { + str += ` for ${ device }`; + } + if ( os ) { + str += ` on ${ os }`; + } + if ( osVersion ) { + str += ` ${ osVersion }`; + } + if ( headless && browserSupportsHeadless( browser ) ) { + str += " (headless)"; + } + return str; +} diff --git a/test/runner/lib/prettyMs.js b/test/runner/lib/prettyMs.js new file mode 100644 index 000000000..99bae2b35 --- /dev/null +++ b/test/runner/lib/prettyMs.js @@ -0,0 +1,18 @@ +/** + * Pretty print a time in milliseconds. + */ +export function prettyMs( time ) { + const minutes = Math.floor( time / 60000 ); + const seconds = Math.floor( time / 1000 ); + const ms = Math.floor( time % 1000 ); + + let prettyTime = `${ ms }ms`; + if ( seconds > 0 ) { + prettyTime = `${ seconds }s ${ prettyTime }`; + } + if ( minutes > 0 ) { + prettyTime = `${ minutes }m ${ prettyTime }`; + } + + return prettyTime; +} diff --git a/test/runner/listeners.js b/test/runner/listeners.js new file mode 100644 index 000000000..a3c52c21e --- /dev/null +++ b/test/runner/listeners.js @@ -0,0 +1,88 @@ +( function() { + + "use strict"; + + // Get the report ID from the URL. + var match = location.search.match( /reportId=([^&]+)/ ); + if ( !match ) { + return; + } + var id = match[ 1 ]; + + // Adopted from https://github.com/douglascrockford/JSON-js + // Support: IE 11+ + // Using the replacer argument of JSON.stringify in IE has issues + // TODO: Replace this with a circular replacer + JSON.stringify + WeakSet + function decycle( object ) { + var objects = []; + + // The derez function recurses through the object, producing the deep copy. + function derez( value ) { + if ( + typeof value === "object" && + value !== null && + !( value instanceof Boolean ) && + !( value instanceof Date ) && + !( value instanceof Number ) && + !( value instanceof RegExp ) && + !( value instanceof String ) + ) { + + // Return a string early for elements + if ( value.nodeType ) { + return value.toString(); + } + + if ( objects.indexOf( value ) > -1 ) { + return; + } + + objects.push( value ); + + if ( Array.isArray( value ) ) { + + // If it is an array, replicate the array. + return value.map( derez ); + } else { + + // If it is an object, replicate the object. + var nu = Object.create( null ); + Object.keys( value ).forEach( function( name ) { + nu[ name ] = derez( value[ name ] ); + } ); + return nu; + } + } + return value; + } + return derez( object ); + } + + function send( type, data ) { + var json = JSON.stringify( { + id: id, + type: type, + data: data ? decycle( data ) : undefined + } ); + var request = new XMLHttpRequest(); + request.open( "POST", "/api/report", true ); + request.setRequestHeader( "Content-Type", "application/json" ); + request.send( json ); + } + + // Send acknowledgement to the server. + send( "ack" ); + + QUnit.on( "testEnd", function( data ) { + send( "testEnd", data ); + } ); + + QUnit.on( "runEnd", function( data ) { + + // Reduce the payload size. + // childSuites is large and unused. + data.childSuites = undefined; + + send( "runEnd", data ); + } ); +} )(); diff --git a/test/runner/modules.js b/test/runner/modules.js new file mode 100644 index 000000000..53f9a933a --- /dev/null +++ b/test/runner/modules.js @@ -0,0 +1,24 @@ +export const modules = [ + "basic", + + "ajax", + "animation", + "attributes", + "callbacks", + "core", + "css", + "data", + "deferred", + "deprecated", + "dimensions", + "effects", + "event", + "manipulation", + "offset", + "queue", + "selector", + "serialize", + "support", + "traversing", + "tween" +]; diff --git a/test/runner/queue.js b/test/runner/queue.js new file mode 100644 index 000000000..4c9d66d8f --- /dev/null +++ b/test/runner/queue.js @@ -0,0 +1,102 @@ +// Build a queue that runs both browsers and modules +// in parallel when the length reaches the concurrency limit +// and refills the queue when one promise resolves. + +import chalk from "chalk"; +import { getAvailableSessions } from "./browserstack/api.js"; +import { runWorker } from "./browserstack/workers.js"; +import { getBrowserString } from "./lib/getBrowserString.js"; +import { runSelenium } from "./selenium/runSelenium.js"; +import { runJSDOM } from "./jsdom.js"; + +const queue = []; +const promises = []; + +const SELENIUM_WAIT_TIME = 100; +const BROWSERSTACK_WAIT_TIME = 5000; +const WORKER_WAIT_TIME = 30000; + +// Limit concurrency to 8 by default in selenium +// BrowserStack defaults to the max allowed by the plan +// More than this will log MaxListenersExceededWarning +const MAX_CONCURRENCY = 8; + +export function addRun( url, browser, options ) { + queue.push( { url, browser, options } ); +} + +export async function runFullQueue( { + browserstack, + concurrency: defaultConcurrency, + verbose +} ) { + while ( queue.length ) { + const next = queue.shift(); + const { url, browser, options } = next; + + const fullBrowser = getBrowserString( browser, options.headless ); + console.log( + `\nRunning ${ chalk.yellow( options.modules.join( ", " ) ) } tests ` + + `in ${ chalk.yellow( fullBrowser ) } (${ chalk.bold( options.reportId ) })...` + ); + + // Wait enough time between requests + // to give concurrency a chance to update. + // In selenium, this helps avoid undici connect timeout errors. + await new Promise( ( resolve ) => + setTimeout( + resolve, + browserstack ? BROWSERSTACK_WAIT_TIME : SELENIUM_WAIT_TIME + ) + ); + + const concurrency = + browserstack && !defaultConcurrency ? + await getAvailableSessions() : + defaultConcurrency || MAX_CONCURRENCY; + + if ( verbose ) { + console.log( + `\nConcurrency: ${ concurrency }. Tests remaining: ${ queue.length + 1 }.` + ); + } + + // If concurrency is 0, wait a bit and try again + if ( concurrency <= 0 ) { + if ( verbose ) { + console.log( "\nWaiting for available sessions..." ); + } + queue.unshift( next ); + await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) ); + continue; + } + + let promise; + if ( browser.browser === "jsdom" ) { + promise = runJSDOM( url, options ); + } else if ( browserstack ) { + promise = runWorker( url, browser, options ); + } else { + promise = runSelenium( url, browser, options ); + } + + // Remove the promise from the list when it resolves + promise.then( () => { + const index = promises.indexOf( promise ); + if ( index !== -1 ) { + promises.splice( index, 1 ); + } + } ); + + // Add the promise to the list + promises.push( promise ); + + // Wait until at least one promise resolves + // if we've reached the concurrency limit + if ( promises.length >= concurrency ) { + await Promise.any( promises ); + } + } + + await Promise.all( promises ); +} diff --git a/test/runner/reporter.js b/test/runner/reporter.js new file mode 100644 index 000000000..bb5e7601b --- /dev/null +++ b/test/runner/reporter.js @@ -0,0 +1,54 @@ +import chalk from "chalk"; +import { getBrowserString } from "./lib/getBrowserString.js"; +import { prettyMs } from "./lib/prettyMs.js"; + +export function reportTest( test, reportId, { browser, headless } ) { + if ( test.status === "passed" ) { + + // Write to console without newlines + process.stdout.write( "." ); + return; + } + + let message = `Test ${ test.status } on ${ chalk.yellow( + getBrowserString( browser, headless ) + ) } (${ chalk.bold( reportId ) }).`; + message += `\n${ chalk.bold( `${ test.suiteName }: ${ test.name }` ) }`; + + // Prefer failed assertions over error messages + if ( test.assertions.filter( ( a ) => !!a && !a.passed ).length ) { + test.assertions.forEach( ( assertion, i ) => { + if ( !assertion.passed ) { + message += `\n${ i + 1 }. ${ chalk.red( assertion.message ) }`; + message += `\n${ chalk.gray( assertion.stack ) }`; + } + } ); + } else if ( test.errors.length ) { + for ( const error of test.errors ) { + message += `\n${ chalk.red( error.message ) }`; + message += `\n${ chalk.gray( error.stack ) }`; + } + } + + console.log( "\n\n" + message ); + + if ( test.status === "failed" ) { + return message; + } +} + +export function reportEnd( result, reportId, { browser, headless, modules } ) { + console.log( + `\n\nTests for ${ chalk.yellow( modules.join( ", " ) ) } on ${ chalk.yellow( + getBrowserString( browser, headless ) + ) } finished in ${ prettyMs( result.runtime ) } (${ chalk.bold( reportId ) }).` + ); + console.log( + ( result.status !== "passed" ? + `${ chalk.red( result.testCounts.failed ) } failed. ` : + "" ) + + `${ chalk.green( result.testCounts.total ) } passed. ` + + `${ chalk.gray( result.testCounts.skipped ) } skipped.` + ); + return result.testCounts; +} diff --git a/test/runner/run.js b/test/runner/run.js new file mode 100644 index 000000000..89b0c5eb7 --- /dev/null +++ b/test/runner/run.js @@ -0,0 +1,315 @@ +import chalk from "chalk"; +import { asyncExitHook, gracefulExit } from "exit-hook"; +import { getLatestBrowser } from "./browserstack/api.js"; +import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js"; +import { localTunnel } from "./browserstack/local.js"; +import { reportEnd, reportTest } from "./reporter.js"; +import { + cleanupAllWorkers, + cleanupWorker, + debugWorker, + retryTest, + touchWorker +} from "./browserstack/workers.js"; +import { createTestServer } from "./createTestServer.js"; +import { buildTestUrl } from "./lib/buildTestUrl.js"; +import { generateHash, printModuleHashes } from "./lib/generateHash.js"; +import { getBrowserString } from "./lib/getBrowserString.js"; +import { addRun, runFullQueue } from "./queue.js"; +import { cleanupAllJSDOM, cleanupJSDOM } from "./jsdom.js"; +import { modules as allModules } from "./modules.js"; + +const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000; + +/** + * Run modules in parallel in different browser instances. + */ +export async function run( { + browsers: browserNames, + browserstack, + concurrency, + debug, + esm, + headless, + isolate = true, + modules = [], + retries = 3, + verbose +} = {} ) { + if ( !browserNames || !browserNames.length ) { + browserNames = [ "chrome" ]; + } + if ( !modules.length ) { + modules = allModules; + } + if ( headless && debug ) { + throw new Error( + "Cannot run in headless mode and debug mode at the same time." + ); + } + + if ( verbose ) { + console.log( browserstack ? "Running in BrowserStack." : "Running locally." ); + } + + const errorMessages = []; + const pendingErrors = {}; + + // Convert browser names to browser objects + let browsers = browserNames.map( ( b ) => ( { browser: b } ) ); + + // A unique identifier for this run + const runId = generateHash( + `${ Date.now() }-${ modules.join( ":" ) }-${ ( browserstack || [] ) + .concat( browserNames ) + .join( ":" ) }` + ); + + // Create the test app and + // hook it up to the reporter + const reports = Object.create( null ); + const app = await createTestServer( async( message ) => { + switch ( message.type ) { + case "testEnd": { + const reportId = message.id; + touchWorker( reportId ); + const errors = reportTest( message.data, reportId, reports[ reportId ] ); + pendingErrors[ reportId ] ||= {}; + if ( errors ) { + pendingErrors[ reportId ][ message.data.name ] = errors; + } else { + delete pendingErrors[ reportId ][ message.data.name ]; + } + break; + } + case "runEnd": { + const reportId = message.id; + const report = reports[ reportId ]; + const { failed, total } = reportEnd( + message.data, + message.id, + reports[ reportId ] + ); + report.total = total; + + if ( failed ) { + if ( !retryTest( reportId, retries ) ) { + if ( debug ) { + debugWorker( reportId ); + } + errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) ); + } + } else { + if ( Object.keys( pendingErrors[ reportId ] ).length ) { + console.warn( "Detected flaky tests:" ); + for ( const [ , error ] in Object.entries( pendingErrors[ reportId ] ) ) { + console.warn( chalk.italic( chalk.gray( error ) ) ); + } + delete pendingErrors[ reportId ]; + } + } + await cleanupWorker( reportId, verbose ); + cleanupJSDOM( reportId, verbose ); + break; + } + case "ack": { + touchWorker( message.id ); + if ( verbose ) { + console.log( `\nWorker for test ${ message.id } acknowledged.` ); + } + break; + } + default: + console.warn( "Received unknown message type:", message.type ); + } + } ); + + // Start up local test server + let server; + let port; + await new Promise( ( resolve ) => { + + // Pass 0 to choose a random, unused port + server = app.listen( 0, () => { + port = server.address().port; + resolve(); + } ); + } ); + + if ( !server || !port ) { + throw new Error( "Server not started." ); + } + + if ( verbose ) { + console.log( `Server started on port ${ port }.` ); + } + + function stopServer() { + return new Promise( ( resolve ) => { + server.close( () => { + if ( verbose ) { + console.log( "Server stopped." ); + } + resolve(); + } ); + } ); + } + + async function cleanup() { + console.log( "Cleaning up..." ); + + if ( tunnel ) { + await tunnel.stop(); + if ( verbose ) { + console.log( "Stopped BrowserStackLocal." ); + } + } + + await cleanupAllWorkers( verbose ); + cleanupAllJSDOM( verbose ); + } + + asyncExitHook( + async() => { + await stopServer(); + await cleanup(); + }, + { wait: EXIT_HOOK_WAIT_TIMEOUT } + ); + + // Start up BrowserStackLocal + let tunnel; + if ( browserstack ) { + if ( headless ) { + console.warn( + chalk.italic( + "BrowserStack does not support headless mode. Running in normal mode." + ) + ); + headless = false; + } + + // Convert browserstack to browser objects. + // If browserstack is an empty array, fall back + // to the browsers array. + if ( browserstack.length ) { + browsers = browserstack.map( ( b ) => { + if ( !b ) { + return browsers[ 0 ]; + } + return buildBrowserFromString( b ); + } ); + } + + // Fill out browser defaults + browsers = await Promise.all( + browsers.map( async( browser ) => { + + // Avoid undici connect timeout errors + await new Promise( ( resolve ) => setTimeout( resolve, 100 ) ); + + const latestMatch = await getLatestBrowser( browser ); + if ( !latestMatch ) { + throw new Error( `Browser not found: ${ getBrowserString( browser ) }.` ); + } + return latestMatch; + } ) + ); + + tunnel = await localTunnel( runId ); + if ( verbose ) { + console.log( "Started BrowserStackLocal." ); + + printModuleHashes( modules ); + } + } + + function queueRun( modules, browser ) { + const fullBrowser = getBrowserString( browser, headless ); + const reportId = generateHash( `${ modules.join( ":" ) } ${ fullBrowser }` ); + reports[ reportId ] = { browser, headless, modules }; + + const url = buildTestUrl( modules, { + browserstack, + esm, + jsdom: browser.browser === "jsdom", + port, + reportId + } ); + + addRun( url, browser, { + debug, + headless, + modules, + reportId, + retries, + runId, + verbose + } ); + } + + for ( const browser of browsers ) { + if ( isolate ) { + for ( const module of modules ) { + queueRun( [ module ], browser ); + } + } else { + queueRun( modules, browser ); + } + } + + try { + console.log( `Starting Run ${ runId }...` ); + await runFullQueue( { browserstack, concurrency, verbose } ); + } catch ( error ) { + console.error( error ); + if ( !debug ) { + gracefulExit( 1 ); + } + } finally { + console.log(); + if ( errorMessages.length === 0 ) { + let stop = false; + for ( const report of Object.values( reports ) ) { + if ( !report.total ) { + stop = true; + console.error( + chalk.red( + `No tests were run for ${ report.modules.join( + ", " + ) } in ${ getBrowserString( report.browser ) }` + ) + ); + } + } + if ( stop ) { + return gracefulExit( 1 ); + } + console.log( chalk.green( "All tests passed!" ) ); + + if ( !debug || browserstack ) { + gracefulExit( 0 ); + } + } else { + console.error( chalk.red( `${ errorMessages.length } tests failed.` ) ); + console.log( + errorMessages.map( ( error, i ) => `\n${ i + 1 }. ${ error }` ).join( "\n" ) + ); + + if ( debug ) { + console.log(); + if ( browserstack ) { + console.log( "Leaving browsers with failures open for debugging." ); + console.log( + "View running sessions at https://automate.browserstack.com/dashboard/v2/" + ); + } else { + console.log( "Leaving browsers open for debugging." ); + } + console.log( "Press Ctrl+C to exit." ); + } else { + gracefulExit( 1 ); + } + } + } +} diff --git a/test/runner/selenium/createDriver.js b/test/runner/selenium/createDriver.js new file mode 100644 index 000000000..765ebb847 --- /dev/null +++ b/test/runner/selenium/createDriver.js @@ -0,0 +1,80 @@ +import { Builder, Capabilities, logging } from "selenium-webdriver"; +import Chrome from "selenium-webdriver/chrome.js"; +import Edge from "selenium-webdriver/edge.js"; +import Firefox from "selenium-webdriver/firefox.js"; +import { browserSupportsHeadless } from "../lib/getBrowserString.js"; + +// Set script timeout to 10min +const DRIVER_SCRIPT_TIMEOUT = 1000 * 60 * 10; + +export default async function createDriver( { browserName, headless, verbose } ) { + const capabilities = Capabilities[ browserName ](); + const prefs = new logging.Preferences(); + prefs.setLevel( logging.Type.BROWSER, logging.Level.ALL ); + capabilities.setLoggingPrefs( prefs ); + + let driver = new Builder().withCapabilities( capabilities ); + + const chromeOptions = new Chrome.Options(); + chromeOptions.addArguments( "--enable-chrome-browser-cloud-management" ); + + // Alter the chrome binary path if + // the CHROME_BIN environment variable is set + if ( process.env.CHROME_BIN ) { + if ( verbose ) { + console.log( `Setting chrome binary to ${ process.env.CHROME_BIN }` ); + } + chromeOptions.setChromeBinaryPath( process.env.CHROME_BIN ); + } + + const firefoxOptions = new Firefox.Options(); + + if ( process.env.FIREFOX_BIN ) { + if ( verbose ) { + console.log( `Setting firefox binary to ${ process.env.FIREFOX_BIN }` ); + } + + firefoxOptions.setBinary( process.env.FIREFOX_BIN ); + } + + const edgeOptions = new Edge.Options(); + edgeOptions.addArguments( "--enable-chrome-browser-cloud-management" ); + + // Alter the edge binary path if + // the EDGE_BIN environment variable is set + if ( process.env.EDGE_BIN ) { + if ( verbose ) { + console.log( `Setting edge binary to ${ process.env.EDGE_BIN }` ); + } + edgeOptions.setEdgeChromiumBinaryPath( process.env.EDGE_BIN ); + } + + if ( headless ) { + chromeOptions.addArguments( "--headless=new" ); + firefoxOptions.addArguments( "--headless" ); + edgeOptions.addArguments( "--headless=new" ); + if ( !browserSupportsHeadless( browserName ) ) { + console.log( + `Headless mode is not supported for ${ browserName }. Running in normal mode instead.` + ); + } + } + + driver = await driver + .setChromeOptions( chromeOptions ) + .setFirefoxOptions( firefoxOptions ) + .setEdgeOptions( edgeOptions ) + .build(); + + if ( verbose ) { + const driverCapabilities = await driver.getCapabilities(); + const name = driverCapabilities.getBrowserName(); + const version = driverCapabilities.getBrowserVersion(); + console.log( `\nDriver created for ${ name } ${ version }` ); + } + + // Increase script timeout to 10min + await driver.manage().setTimeouts( { script: DRIVER_SCRIPT_TIMEOUT } ); + + return driver; +} diff --git a/test/runner/selenium/runSelenium.js b/test/runner/selenium/runSelenium.js new file mode 100644 index 000000000..247cd8472 --- /dev/null +++ b/test/runner/selenium/runSelenium.js @@ -0,0 +1,31 @@ +import chalk from "chalk"; +import createDriver from "./createDriver.js"; + +export async function runSelenium( + url, + { browser }, + { debug, headless, verbose } = {} +) { + if ( debug && headless ) { + throw new Error( "Cannot debug in headless mode." ); + } + + const driver = await createDriver( { + browserName: browser, + headless, + verbose + } ); + + try { + await driver.get( url ); + await driver.executeScript( +`return new Promise( ( resolve ) => { + QUnit.on( "runEnd", resolve ); +} )` + ); + } finally { + if ( !debug || headless ) { + await driver.quit(); + } + } +} diff --git a/test/runner/server.js b/test/runner/server.js new file mode 100644 index 000000000..09fe0da4c --- /dev/null +++ b/test/runner/server.js @@ -0,0 +1,13 @@ +import { createTestServer } from "./createTestServer.js"; + +const port = process.env.PORT || 3000; + +async function runServer() { + const app = await createTestServer(); + + app.listen( { port, host: "0.0.0.0" }, function() { + console.log( `Open tests at http://localhost:${ port }/test/` ); + } ); +} + +runServer(); |