node_modules
.sizecache.json
package-lock.json
+local.log
},
"devDependencies": {
"body-parser": "1.20.2",
+ "browserstack-local": "1.5.5",
"commitplease": "3.2.0",
"diff": "5.2.0",
"eslint-config-jquery": "3.0.2",
{
"files": ["**/*"],
"env": {
+ "es6": true,
"node": true
},
- "globals": {
- "fetch": false,
- "Promise": false,
- "require": false
- },
"parserOptions": {
"ecmaVersion": 2022,
"sourceType": "module"
},
"globals": {
"QUnit": false,
- "Symbol": false
+ "Symbol": false,
+ "require": false
},
"parserOptions": {
"ecmaVersion": 5,
-// This list is static, so no requests are required
-// in the command help menu.
+import chalk from "chalk";
+import { getBrowserString } from "./lib/getBrowserString.js";
+import {
+ createWorker,
+ deleteWorker,
+ getAvailableSessions
+} from "./browserstack/api.js";
+import createDriver from "./selenium/createDriver.js";
-export const browsers = [ "chrome", "ie", "firefox", "edge", "safari", "opera" ];
+const workers = Object.create( null );
+
+/**
+ * Keys are browser strings
+ * Structure of a worker:
+ * {
+ * browser: object // The browser object
+ * debug: boolean // Stops the worker from being cleaned up when finished
+ * lastTouch: number // The last time a request was received
+ * restarts: number // The number of times the worker has been restarted
+ * options: object // The options to create the worker
+ * url: string // The URL the worker is on
+ * quit: function // A function to stop the worker
+ * }
+ */
+
+// Acknowledge the worker within the time limit.
+// BrowserStack can take much longer spinning up
+// some browsers, such as iOS 15 Safari.
+const ACKNOWLEDGE_INTERVAL = 1000;
+const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 5;
+
+const MAX_WORKER_RESTARTS = 5;
+
+// No report after the time limit
+// should refresh the worker
+const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
+
+const WORKER_WAIT_TIME = 30000;
+
+// Limit concurrency to 8 by default in selenium
+const MAX_SELENIUM_CONCURRENCY = 8;
+
+export async function createBrowserWorker( url, browser, options, restarts = 0 ) {
+ if ( restarts > MAX_WORKER_RESTARTS ) {
+ throw new Error(
+ `Reached the maximum number of restarts for ${ chalk.yellow(
+ getBrowserString( browser )
+ ) }`
+ );
+ }
+ const { browserstack, debug, headless, runId, tunnelId, verbose } = options;
+ while ( await maxWorkersReached( options ) ) {
+ if ( verbose ) {
+ console.log( "\nWaiting for available sessions..." );
+ }
+ await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
+ }
+
+ const fullBrowser = getBrowserString( browser );
+
+ let worker;
+
+ if ( browserstack ) {
+ worker = await createWorker( {
+ ...browser,
+ url: encodeURI( url ),
+ project: "jquery",
+ build: `Run ${ runId }`,
+
+ // This is the maximum timeout allowed
+ // by BrowserStack. We do this because
+ // we control the timeout in the runner.
+ // See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300
+ timeout: 1800,
+
+ // Not documented in the API docs,
+ // but required to make local testing work.
+ // See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
+ "browserstack.local": true,
+ "browserstack.localIdentifier": tunnelId
+ } );
+ worker.quit = () => deleteWorker( worker.id );
+ } else {
+ const driver = await createDriver( {
+ browserName: browser.browser,
+ headless,
+ url,
+ verbose
+ } );
+ worker = {
+ quit: () => driver.quit()
+ };
+ }
+
+ worker.debug = !!debug;
+ worker.url = url;
+ worker.browser = browser;
+ worker.restarts = restarts;
+ worker.options = options;
+ touchBrowser( browser );
+ workers[ fullBrowser ] = worker;
+
+ // Wait for the worker to show up in the list
+ // before returning it.
+ return ensureAcknowledged( worker );
+}
+
+export function touchBrowser( browser ) {
+ const fullBrowser = getBrowserString( browser );
+ const worker = workers[ fullBrowser ];
+ if ( worker ) {
+ worker.lastTouch = Date.now();
+ }
+}
+
+export async function setBrowserWorkerUrl( browser, url ) {
+ const fullBrowser = getBrowserString( browser );
+ const worker = workers[ fullBrowser ];
+ if ( worker ) {
+ worker.url = url;
+ }
+}
+
+export async function restartBrowser( browser ) {
+ const fullBrowser = getBrowserString( browser );
+ const worker = workers[ fullBrowser ];
+ if ( worker ) {
+ await restartWorker( worker );
+ }
+}
+
+/**
+ * Checks that all browsers have received
+ * a response in the given amount of time.
+ * If not, the worker is restarted.
+ */
+export async function checkLastTouches() {
+ for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) {
+ if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) {
+ const options = worker.options;
+ if ( options.verbose ) {
+ console.log(
+ `\nNo response from ${ chalk.yellow( fullBrowser ) } in ${
+ RUN_WORKER_TIMEOUT / 1000 / 60
+ }min.`
+ );
+ }
+ await restartWorker( worker );
+ }
+ }
+}
+
+export async function cleanupAllBrowsers( { verbose } ) {
+ const workersRemaining = Object.values( workers );
+ const numRemaining = workersRemaining.length;
+ if ( numRemaining ) {
+ try {
+ await Promise.all( workersRemaining.map( ( worker ) => worker.quit() ) );
+ if ( verbose ) {
+ console.log(
+ `Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
+ );
+ }
+ } catch ( error ) {
+
+ // Log the error, but do not consider the test run failed
+ console.error( error );
+ }
+ }
+}
+
+async function maxWorkersReached( {
+ browserstack,
+ concurrency = MAX_SELENIUM_CONCURRENCY
+} ) {
+ if ( browserstack ) {
+ return ( await getAvailableSessions() ) <= 0;
+ }
+ return workers.length >= concurrency;
+}
+
+async function waitForAck( worker, { fullBrowser, verbose } ) {
+ delete worker.lastTouch;
+ return new Promise( ( resolve, reject ) => {
+ const interval = setInterval( () => {
+ if ( worker.lastTouch ) {
+ if ( verbose ) {
+ console.log( `\n${ fullBrowser } acknowledged.` );
+ }
+ clearTimeout( timeout );
+ clearInterval( interval );
+ resolve();
+ }
+ }, ACKNOWLEDGE_INTERVAL );
+
+ const timeout = setTimeout( () => {
+ clearInterval( interval );
+ reject(
+ new Error(
+ `${ fullBrowser } not acknowledged after ${
+ ACKNOWLEDGE_TIMEOUT / 1000 / 60
+ }min.`
+ )
+ );
+ }, ACKNOWLEDGE_TIMEOUT );
+ } );
+}
+
+async function ensureAcknowledged( worker ) {
+ const fullBrowser = getBrowserString( worker.browser );
+ const verbose = worker.options.verbose;
+ try {
+ await waitForAck( worker, { fullBrowser, verbose } );
+ return worker;
+ } catch ( error ) {
+ console.error( error.message );
+ await restartWorker( worker );
+ }
+}
+
+async function cleanupWorker( worker, { verbose } ) {
+ for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
+ if ( w === worker ) {
+ delete workers[ fullBrowser ];
+ await worker.quit();
+ if ( verbose ) {
+ console.log( `\nStopped ${ fullBrowser }.` );
+ }
+ return;
+ }
+ }
+}
+
+async function restartWorker( worker ) {
+ await cleanupWorker( worker, worker.options );
+ await createBrowserWorker(
+ worker.url,
+ worker.browser,
+ worker.options,
+ worker.restarts + 1
+ );
+}
--- /dev/null
+/**
+ * Browserstack API is documented at
+ * https://github.com/browserstack/api
+ */
+
+import { createAuthHeader } from "./createAuthHeader.js";
+
+const browserstackApi = "https://api.browserstack.com";
+const apiVersion = 5;
+
+const username = process.env.BROWSERSTACK_USERNAME;
+const accessKey = process.env.BROWSERSTACK_ACCESS_KEY;
+
+// iOS has null for version numbers,
+// and we do not need a similar check for OS versions.
+const rfinalVersion = /(?:^[0-9\.]+$)|(?:^null$)/;
+const rlatest = /^latest-(\d+)$/;
+
+const rnonDigits = /(?:[^\d\.]+)|(?:20\d{2})/g;
+
+async function fetchAPI( path, options = {}, versioned = true ) {
+ if ( !username || !accessKey ) {
+ throw new Error(
+ "BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables must be set."
+ );
+ }
+ const init = {
+ method: "GET",
+ ...options,
+ headers: {
+ authorization: createAuthHeader( username, accessKey ),
+ accept: "application/json",
+ "content-type": "application/json",
+ ...options.headers
+ }
+ };
+ const response = await fetch(
+ `${ browserstackApi }/${ versioned ? `${ apiVersion }/` : "" }${ path }`,
+ init
+ );
+ if ( !response.ok ) {
+ console.log(
+ `\n${ init.method } ${ path }`,
+ response.status,
+ response.statusText
+ );
+ throw new Error( `Error fetching ${ path }` );
+ }
+ return response.json();
+}
+
+/**
+ * =============================
+ * Browsers API
+ * =============================
+ */
+
+function compareVersionNumbers( a, b ) {
+ if ( a != null && b == null ) {
+ return -1;
+ }
+ if ( a == null && b != null ) {
+ return 1;
+ }
+ if ( a == null && b == null ) {
+ return 0;
+ }
+ const aParts = a.replace( rnonDigits, "" ).split( "." );
+ const bParts = b.replace( rnonDigits, "" ).split( "." );
+
+ if ( aParts.length > bParts.length ) {
+ return -1;
+ }
+ if ( aParts.length < bParts.length ) {
+ return 1;
+ }
+
+ for ( let i = 0; i < aParts.length; i++ ) {
+ const aPart = Number( aParts[ i ] );
+ const bPart = Number( bParts[ i ] );
+ if ( aPart < bPart ) {
+ return -1;
+ }
+ if ( aPart > bPart ) {
+ return 1;
+ }
+ }
+
+ if ( rnonDigits.test( a ) && !rnonDigits.test( b ) ) {
+ return -1;
+ }
+ if ( !rnonDigits.test( a ) && rnonDigits.test( b ) ) {
+ return 1;
+ }
+
+ return 0;
+}
+
+function sortBrowsers( a, b ) {
+ if ( a.browser < b.browser ) {
+ return -1;
+ }
+ if ( a.browser > b.browser ) {
+ return 1;
+ }
+ const browserComparison = compareVersionNumbers(
+ a.browser_version,
+ b.browser_version
+ );
+ if ( browserComparison ) {
+ return browserComparison;
+ }
+ if ( a.os < b.os ) {
+ return -1;
+ }
+ if ( a.os > b.os ) {
+ return 1;
+ }
+ const osComparison = compareVersionNumbers( a.os_version, b.os_version );
+ if ( osComparison ) {
+ return osComparison;
+ }
+ const deviceComparison = compareVersionNumbers( a.device, b.device );
+ if ( deviceComparison ) {
+ return deviceComparison;
+ }
+ return 0;
+}
+
+export async function getBrowsers( { flat = false } = {} ) {
+ const query = new URLSearchParams();
+ if ( flat ) {
+ query.append( "flat", true );
+ }
+ const browsers = await fetchAPI( `/browsers?${ query }` );
+ return browsers.sort( sortBrowsers );
+}
+
+function matchVersion( browserVersion, version ) {
+ if ( !version ) {
+ return false;
+ }
+ const regex = new RegExp(
+ `^${ version.replace( /\\/g, "\\\\" ).replace( /\./g, "\\." ) }\\b`,
+ "i"
+ );
+ return regex.test( browserVersion );
+}
+
+export async function filterBrowsers( filter ) {
+ const browsers = await getBrowsers( { flat: true } );
+ if ( !filter ) {
+ return browsers;
+ }
+ const filterBrowser = ( filter.browser ?? "" ).toLowerCase();
+ const filterVersion = ( filter.browser_version ?? "" ).toLowerCase();
+ const filterOs = ( filter.os ?? "" ).toLowerCase();
+ const filterOsVersion = ( filter.os_version ?? "" ).toLowerCase();
+ const filterDevice = ( filter.device ?? "" ).toLowerCase();
+
+ const filteredWithoutVersion = browsers.filter( ( browser ) => {
+ return (
+ ( !filterBrowser || filterBrowser === browser.browser.toLowerCase() ) &&
+ ( !filterOs || filterOs === browser.os.toLowerCase() ) &&
+ ( !filterOsVersion || matchVersion( browser.os_version, filterOsVersion ) ) &&
+ ( !filterDevice || filterDevice === ( browser.device || "" ).toLowerCase() )
+ );
+ } );
+
+ if ( !filterVersion ) {
+ return filteredWithoutVersion;
+ }
+
+ if ( filterVersion.startsWith( "latest" ) ) {
+ const groupedByName = filteredWithoutVersion
+ .filter( ( b ) => rfinalVersion.test( b.browser_version ) )
+ .reduce( ( acc, browser ) => {
+ acc[ browser.browser ] = acc[ browser.browser ] ?? [];
+ acc[ browser.browser ].push( browser );
+ return acc;
+ }, Object.create( null ) );
+
+ const filtered = [];
+ for ( const group of Object.values( groupedByName ) ) {
+ const latest = group[ group.length - 1 ];
+
+ // Mobile devices do not have browser version.
+ // Skip the version check for these,
+ // but include the latest in the list if it made it
+ // through filtering.
+ if ( !latest.browser_version ) {
+
+ // Do not include in the list for latest-n.
+ if ( filterVersion === "latest" ) {
+ filtered.push( latest );
+ }
+ continue;
+ }
+
+ // Get the latest version and subtract the number from the filter,
+ // ignoring any patch versions, which may differ between major versions.
+ const num = rlatest.exec( filterVersion );
+ const version = parseInt( latest.browser_version ) - ( num ? num[ 1 ] : 0 );
+ const match = group.findLast( ( browser ) => {
+ return matchVersion( browser.browser_version, version.toString() );
+ } );
+ if ( match ) {
+ filtered.push( match );
+ }
+ }
+ return filtered;
+ }
+
+ return filteredWithoutVersion.filter( ( browser ) => {
+ return matchVersion( browser.browser_version, filterVersion );
+ } );
+}
+
+export async function listBrowsers( filter ) {
+ const browsers = await filterBrowsers( filter );
+ console.log( "Available browsers:" );
+ for ( const browser of browsers ) {
+ let message = ` ${ browser.browser }_`;
+ if ( browser.device ) {
+ message += `:${ browser.device }_`;
+ } else {
+ message += `${ browser.browser_version }_`;
+ }
+ message += `${ browser.os }_${ browser.os_version }`;
+ console.log( message );
+ }
+}
+
+export async function getLatestBrowser( filter ) {
+ if ( !filter.browser_version ) {
+ filter.browser_version = "latest";
+ }
+ const browsers = await filterBrowsers( filter );
+ return browsers[ browsers.length - 1 ];
+}
+
+/**
+ * =============================
+ * Workers API
+ * =============================
+ */
+
+/**
+ * A browser object may only have one of `browser` or `device` set;
+ * which property is set will depend on `os`.
+ *
+ * `options`: is an object with the following properties:
+ * `os`: The operating system.
+ * `os_version`: The operating system version.
+ * `browser`: The browser name.
+ * `browser_version`: The browser version.
+ * `device`: The device name.
+ * `url` (optional): Which URL to navigate to upon creation.
+ * `timeout` (optional): Maximum life of the worker (in seconds). Maximum value of `1800`. Specifying `0` will use the default of `300`.
+ * `name` (optional): Provide a name for the worker.
+ * `build` (optional): Group workers into a build.
+ * `project` (optional): Provide the project the worker belongs to.
+ * `resolution` (optional): Specify the screen resolution (e.g. "1024x768").
+ * `browserstack.local` (optional): Set to `true` to mark as local testing.
+ * `browserstack.video` (optional): Set to `false` to disable video recording.
+ * `browserstack.localIdentifier` (optional): ID of the local tunnel.
+ */
+export function createWorker( options ) {
+ return fetchAPI( "/worker", {
+ method: "POST",
+ body: JSON.stringify( options )
+ } );
+}
+
+/**
+ * Returns a worker object, if one exists, with the following properties:
+ * `id`: The worker id.
+ * `status`: A string representing the current status of the worker.
+ * Possible statuses: `"running"`, `"queue"`.
+ */
+export function getWorker( id ) {
+ return fetchAPI( `/worker/${ id }` );
+}
+
+export async function deleteWorker( id ) {
+ return fetchAPI( `/worker/${ id }`, { method: "DELETE" } );
+}
+
+export function getWorkers() {
+ return fetchAPI( "/workers" );
+}
+
+/**
+ * Stop all workers
+ */
+export async function stopWorkers() {
+ const workers = await getWorkers();
+
+ // Run each request on its own
+ // to avoid connect timeout errors.
+ console.log( `${ workers.length } workers running...` );
+ for ( const worker of workers ) {
+ try {
+ await deleteWorker( worker.id );
+ } catch ( error ) {
+
+ // Log the error, but continue trying to remove workers.
+ console.error( error );
+ }
+ }
+ console.log( "All workers stopped." );
+}
+
+/**
+ * =============================
+ * Plan API
+ * =============================
+ */
+
+export function getPlan() {
+ return fetchAPI( "/automate/plan.json", {}, false );
+}
+
+export async function getAvailableSessions() {
+ try {
+ const [ plan, workers ] = await Promise.all( [ getPlan(), getWorkers() ] );
+ return plan.parallel_sessions_max_allowed - workers.length;
+ } catch ( error ) {
+ console.error( error );
+ return 0;
+ }
+}
--- /dev/null
+export function buildBrowserFromString( str ) {
+ const [ browser, versionOrDevice, os, osVersion ] = str.split( "_" );
+
+ // If the version starts with a colon, it's a device
+ if ( versionOrDevice && versionOrDevice.startsWith( ":" ) ) {
+ return {
+ browser,
+ device: versionOrDevice.slice( 1 ),
+ os,
+ os_version: osVersion
+ };
+ }
+
+ return {
+ browser,
+ browser_version: versionOrDevice,
+ os,
+ os_version: osVersion
+ };
+}
--- /dev/null
+const textEncoder = new TextEncoder();
+
+export function createAuthHeader( username, accessKey ) {
+ const encoded = textEncoder.encode( `${ username }:${ accessKey }` );
+ const base64 = btoa( String.fromCodePoint.apply( null, encoded ) );
+ return `Basic ${ base64 }`;
+}
--- /dev/null
+import browserstackLocal from "browserstack-local";
+
+export async function localTunnel( localIdentifier, opts = {} ) {
+ const tunnel = new browserstackLocal.Local();
+
+ return new Promise( ( resolve, reject ) => {
+
+ // https://www.browserstack.com/docs/local-testing/binary-params
+ tunnel.start(
+ {
+ "enable-logging-for-api": "",
+ localIdentifier,
+ ...opts
+ },
+ async( error ) => {
+ if ( error || !tunnel.isRunning() ) {
+ return reject( error );
+ }
+ resolve( {
+ stop: function stopTunnel() {
+ return new Promise( ( resolve, reject ) => {
+ tunnel.stop( ( error ) => {
+ if ( error ) {
+ return reject( error );
+ }
+ resolve();
+ } );
+ } );
+ }
+ } );
+ }
+ );
+ } );
+}
import yargs from "yargs/yargs";
-import { browsers } from "./browsers.js";
-import { suites } from "./suites.js";
+import { browsers } from "./flags/browsers.js";
+import { getPlan, listBrowsers, stopWorkers } from "./browserstack/api.js";
+import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js";
+import { jquery } from "./flags/jquery.js";
+import { suites } from "./flags/suites.js";
import { run } from "./run.js";
-import { jquery } from "./jquery.js";
const argv = yargs( process.argv.slice( 2 ) )
.version( false )
type: "array",
choices: browsers,
description:
- "Run tests in a specific browser.\n" +
- "Pass multiple browsers by repeating the option.",
+ "Run tests in a specific browser." +
+ "Pass multiple browsers by repeating the option." +
+ "If using BrowserStack, specify browsers using --browserstack.",
default: [ "chrome" ]
} )
.option( "headless", {
alias: "h",
type: "boolean",
description:
- "Run tests in headless mode. Cannot be used with --debug.",
- conflicts: [ "debug" ]
+ "Run tests in headless mode. Cannot be used with --debug or --browserstack.",
+ conflicts: [ "debug", "browserstack" ]
+ } )
+ .option( "concurrency", {
+ alias: "c",
+ type: "number",
+ description:
+ "Run tests in parallel in multiple browsers. " +
+ "Defaults to 8 in normal mode. In browserstack mode, " +
+ "defaults to the maximum available under your BrowserStack plan."
} )
.option( "debug", {
alias: "d",
.option( "retries", {
alias: "r",
type: "number",
- description: "Number of times to retry failed tests."
+ description: "Number of times to retry failed tests by refreshing the URL."
} )
- .option( "concurrency", {
- alias: "c",
+ .option( "hard-retries", {
type: "number",
- description: "Run tests in parallel in multiple browsers. Defaults to 8."
+ description:
+ "Number of times to retry failed tests by restarting the worker. " +
+ "This is in addition to the normal retries " +
+ "and are only used when the normal retries are exhausted."
} )
.option( "verbose", {
alias: "v",
type: "boolean",
description: "Log additional information."
} )
+ .option( "browserstack", {
+ type: "array",
+ description:
+ "Run tests in BrowserStack.\n" +
+ "Requires BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables.\n" +
+ "The value can be empty for the default configuration, or a string in the format of\n" +
+ "\"browser_[browserVersion | :device]_os_osVersion\" (see --list-browsers).\n" +
+ "Pass multiple browsers by repeating the option.\n" +
+ "The --browser option is ignored when --browserstack has a value.\n" +
+ "Otherwise, the --browser option will be used, " +
+ "with the latest version/device for that browser, on a matching OS."
+ } )
+ .option( "run-id", {
+ type: "string",
+ description: "A unique identifier for the run in BrowserStack."
+ } )
+ .option( "list-browsers", {
+ type: "string",
+ description:
+ "List available BrowserStack browsers and exit.\n" +
+ "Leave blank to view all browsers or pass " +
+ "\"browser_[browserVersion | :device]_os_osVersion\" with each parameter " +
+ "separated by an underscore to filter the list (any can be omitted).\n" +
+ "\"latest\" can be used in place of \"browserVersion\" to find the latest version.\n" +
+ "\"latest-n\" can be used to find the nth latest browser version.\n" +
+ "Use a colon to indicate a device.\n" +
+ "Examples: \"chrome__windows_10\", \"safari_latest\", " +
+ "\"Mobile Safari\", \"Android Browser_:Google Pixel 8 Pro\".\n" +
+ "Use quotes if spaces are necessary."
+ } )
+ .option( "stop-workers", {
+ type: "boolean",
+ description:
+ "WARNING: This will stop all BrowserStack workers that may exist and exit," +
+ "including any workers running from other projects.\n" +
+ "This can be used as a failsafe when there are too many stray workers."
+ } )
+ .option( "browserstack-plan", {
+ type: "boolean",
+ description: "Show BrowserStack plan information and exit."
+ } )
.help().argv;
-run( argv );
+if ( typeof argv.listBrowsers === "string" ) {
+ listBrowsers( buildBrowserFromString( argv.listBrowsers ) );
+} else if ( argv.stopWorkers ) {
+ stopWorkers();
+} else if ( argv.browserstackPlan ) {
+ console.log( await getPlan() );
+} else {
+ run( argv );
+}
+import { readFile } from "node:fs/promises";
import bodyParser from "body-parser";
import express from "express";
import bodyParserErrorHandler from "express-body-parser-error-handler";
-import { readFile } from "node:fs/promises";
export async function createTestServer( report ) {
const app = express();
--- /dev/null
+// This list is static, so no requests are required
+// in the command help menu.
+
+import { getBrowsers } from "../browserstack/api.js";
+
+export const browsers = [
+ "chrome",
+ "ie",
+ "firefox",
+ "edge",
+ "safari",
+ "opera",
+ "yandex",
+ "IE Mobile",
+ "Android Browser",
+ "Mobile Safari"
+];
+
+// A function that can be used to update the above list.
+export async function getAvailableBrowsers() {
+ const browsers = await getBrowsers( { flat: true } );
+ const available = [ ...new Set( browsers.map( ( { browser } ) => browser ) ) ];
+ return available;
+}
--- /dev/null
+// Keep in sync with tests/lib/qunit.js
+export const jquery = [
+ "1.8.0", "1.8.1", "1.8.2", "1.8.3",
+ "1.9.0", "1.9.1",
+ "1.10.0", "1.10.1", "1.10.2",
+ "1.11.0", "1.11.1", "1.11.2", "1.11.3",
+ "1.12.0", "1.12.1", "1.12.2", "1.12.3", "1.12.4",
+ "2.0.0", "2.0.1", "2.0.2", "2.0.3",
+ "2.1.0", "2.1.1", "2.1.2", "2.1.3", "2.1.4",
+ "2.2.0", "2.2.1", "2.2.2", "2.2.3", "2.2.4",
+ "3.0.0",
+ "3.1.0", "3.1.1",
+ "3.2.0", "3.2.1",
+ "3.3.0", "3.3.1",
+ "3.4.0", "3.4.1",
+ "3.5.0", "3.5.1",
+ "3.6.0", "3.6.1", "3.6.2", "3.6.3", "3.6.4",
+ "3.7.0", "3.7.1",
+ "3.x-git", "git", "custom"
+];
--- /dev/null
+export const suites = [
+ "accordion",
+ "autocomplete",
+ "button",
+ "checkboxradio",
+ "controlgroup",
+ "core",
+ "datepicker",
+ "dialog",
+ "draggable",
+ "droppable",
+ "effects",
+ "form-reset-mixin",
+ "menu",
+ "position",
+ "progressbar",
+ "resizable",
+ "selectable",
+ "selectmenu",
+ "slider",
+ "sortable",
+ "spinner",
+ "tabs",
+ "tooltip",
+ "widget"
+];
+++ /dev/null
-// Keep in sync with tests/lib/qunit.js
-export const jquery = [
- "1.8.0", "1.8.1", "1.8.2", "1.8.3",
- "1.9.0", "1.9.1",
- "1.10.0", "1.10.1", "1.10.2",
- "1.11.0", "1.11.1", "1.11.2", "1.11.3",
- "1.12.0", "1.12.1", "1.12.2", "1.12.3", "1.12.4",
- "2.0.0", "2.0.1", "2.0.2", "2.0.3",
- "2.1.0", "2.1.1", "2.1.2", "2.1.3", "2.1.4",
- "2.2.0", "2.2.1", "2.2.2", "2.2.3", "2.2.4",
- "3.0.0",
- "3.1.0", "3.1.1",
- "3.2.0", "3.2.1",
- "3.3.0", "3.3.1",
- "3.4.0", "3.4.1",
- "3.5.0", "3.5.1",
- "3.6.0", "3.6.1", "3.6.2", "3.6.3", "3.6.4",
- "3.7.0", "3.7.1",
- "3.x-git", "git", "custom"
-];
-export function buildTestUrl( suite, { jquery, migrate, port, reportId } ) {
+export function buildTestUrl( suite, { browserstack, jquery, migrate, port, reportId } ) {
if ( !port ) {
throw new Error( "No port specified." );
}
query.append( "reportId", reportId );
}
- return `http://localhost:${ port }/tests/unit/${ suite }/${ suite }.html?${ query }`;
+ // BrowserStack supplies a custom domain for local testing,
+ // which is especially necessary for iOS testing.
+ const host = browserstack ? "bs-local.com" : "localhost";
+ return `http://${ host }:${ port }/tests/unit/${ suite }/${ suite }.html?${ query }`;
}
edge: "Edge",
firefox: "Firefox",
ie: "IE",
- jsdom: "JSDOM",
opera: "Opera",
safari: "Safari"
};
--- /dev/null
+import chalk from "chalk";
+import { getBrowserString } from "./lib/getBrowserString.js";
+import {
+ checkLastTouches,
+ createBrowserWorker,
+ restartBrowser,
+ setBrowserWorkerUrl
+} from "./browsers.js";
+
+const TEST_POLL_TIMEOUT = 1000;
+
+const queue = [];
+
+export function getNextBrowserTest( reportId ) {
+ const index = queue.findIndex( ( test ) => test.id === reportId );
+ if ( index === -1 ) {
+ return;
+ }
+
+ // Remove the completed test from the queue
+ const previousTest = queue[ index ];
+ queue.splice( index, 1 );
+
+ // Find the next test for the same browser
+ for ( const test of queue.slice( index ) ) {
+ if ( test.fullBrowser === previousTest.fullBrowser ) {
+
+ // Set the URL for our tracking
+ setBrowserWorkerUrl( test.browser, test.url );
+ test.running = true;
+
+ // Return the URL for the next test.
+ // listeners.js will use this to set the browser URL.
+ return { url: test.url };
+ }
+ }
+}
+
+export function retryTest( reportId, maxRetries ) {
+ if ( !maxRetries ) {
+ return;
+ }
+ const test = queue.find( ( test ) => test.id === reportId );
+ if ( test ) {
+ test.retries++;
+ if ( test.retries <= maxRetries ) {
+ console.log(
+ `\nRetrying test ${ reportId } for ${ chalk.yellow(
+ test.options.suite
+ ) }...${ test.retries }`
+ );
+ return test;
+ }
+ }
+}
+
+export async function hardRetryTest( reportId, maxHardRetries ) {
+ if ( !maxHardRetries ) {
+ return false;
+ }
+ const test = queue.find( ( test ) => test.id === reportId );
+ if ( test ) {
+ test.hardRetries++;
+ if ( test.hardRetries <= maxHardRetries ) {
+ console.log(
+ `\nHard retrying test ${ reportId } for ${ chalk.yellow(
+ test.options.suite
+ ) }...${ test.hardRetries }`
+ );
+ await restartBrowser( test.browser );
+ return true;
+ }
+ }
+ return false;
+}
+
+export function addRun( url, browser, options ) {
+ queue.push( {
+ browser,
+ fullBrowser: getBrowserString( browser ),
+ hardRetries: 0,
+ id: options.reportId,
+ url,
+ options,
+ retries: 0,
+ running: false
+ } );
+}
+
+export async function runAll() {
+ return new Promise( async( resolve, reject ) => {
+ while ( queue.length ) {
+ try {
+ await checkLastTouches();
+ } catch ( error ) {
+ reject( error );
+ }
+
+ // Run one test URL per browser at a time
+ const browsersTaken = [];
+ for ( const test of queue ) {
+ if ( browsersTaken.indexOf( test.fullBrowser ) > -1 ) {
+ continue;
+ }
+ browsersTaken.push( test.fullBrowser );
+ if ( !test.running ) {
+ test.running = true;
+ try {
+ await createBrowserWorker( test.url, test.browser, test.options );
+ } catch ( error ) {
+ reject( error );
+ }
+ }
+ }
+ await new Promise( ( resolve ) => setTimeout( resolve, TEST_POLL_TIMEOUT ) );
+ }
+ resolve();
+ } );
+}
import chalk from "chalk";
+import * as Diff from "diff";
import { getBrowserString } from "./lib/getBrowserString.js";
import { prettyMs } from "./lib/prettyMs.js";
-import * as Diff from "diff";
function serializeForDiff( value ) {
import chalk from "chalk";
import { asyncExitHook, gracefulExit } from "exit-hook";
+import { getLatestBrowser } from "./browserstack/api.js";
+import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js";
+import { localTunnel } from "./browserstack/local.js";
import { reportEnd, reportTest } from "./reporter.js";
import { createTestServer } from "./createTestServer.js";
import { buildTestUrl } from "./lib/buildTestUrl.js";
import { generateHash } from "./lib/generateHash.js";
import { getBrowserString } from "./lib/getBrowserString.js";
-import { suites as allSuites } from "./suites.js";
-import { cleanupAllBrowsers, touchBrowser } from "./selenium/browsers.js";
-import { addRun, getNextBrowserTest, retryTest, runAll } from "./selenium/queue.js";
+import { suites as allSuites } from "./flags/suites.js";
+import { cleanupAllBrowsers, touchBrowser } from "./browsers.js";
+import {
+ addRun,
+ getNextBrowserTest,
+ hardRetryTest,
+ retryTest,
+ runAll
+} from "./queue.js";
const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
*/
export async function run( {
browser: browserNames = [],
+ browserstack,
concurrency,
debug,
+ hardRetries,
headless,
jquery: jquerys = [],
migrate,
retries = 0,
+ runId,
suite: suites = [],
verbose
} ) {
);
}
+ if ( verbose ) {
+ console.log( browserstack ? "Running in BrowserStack." : "Running locally." );
+ }
+
const errorMessages = [];
const pendingErrors = {};
// Convert browser names to browser objects
let browsers = browserNames.map( ( b ) => ( { browser: b } ) );
+ const tunnelId = generateHash(
+ `${ Date.now() }-${ suites.join( ":" ) }-${ ( browserstack || [] )
+ .concat( browserNames )
+ .join( ":" ) }`
+ );
+
+ // A unique identifier for this run
+ if ( !runId ) {
+ runId = tunnelId;
+ }
// Create the test app and
// hook it up to the reporter
return retry;
}
+ // Return early if hardRetryTest returns true
+ if ( await hardRetryTest( reportId, hardRetries ) ) {
+ return;
+ }
errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
}
} );
}
+ async function cleanup() {
+ console.log( "Cleaning up..." );
+
+ await cleanupAllBrowsers( { verbose } );
+
+ if ( tunnel ) {
+ await tunnel.stop();
+ if ( verbose ) {
+ console.log( "Stopped BrowserStackLocal." );
+ }
+ }
+ }
+
asyncExitHook(
async() => {
- await cleanupAllBrowsers( { verbose } );
+ await cleanup();
await stopServer();
},
{ wait: EXIT_HOOK_WAIT_TIMEOUT }
);
+ // Start up BrowserStackLocal
+ let tunnel;
+ if ( browserstack ) {
+ if ( headless ) {
+ console.warn(
+ chalk.italic(
+ "BrowserStack does not support headless mode. Running in normal mode."
+ )
+ );
+ headless = false;
+ }
+
+ // Convert browserstack to browser objects.
+ // If browserstack is an empty array, fall back
+ // to the browsers array.
+ if ( browserstack.length ) {
+ browsers = browserstack.map( ( b ) => {
+ if ( !b ) {
+ return browsers[ 0 ];
+ }
+ return buildBrowserFromString( b );
+ } );
+ }
+
+ // Fill out browser defaults
+ browsers = await Promise.all(
+ browsers.map( async( browser ) => {
+
+ // Avoid undici connect timeout errors
+ await new Promise( ( resolve ) => setTimeout( resolve, 100 ) );
+
+ const latestMatch = await getLatestBrowser( browser );
+ if ( !latestMatch ) {
+ console.error(
+ chalk.red( `Browser not found: ${ getBrowserString( browser ) }.` )
+ );
+ gracefulExit( 1 );
+ }
+ return latestMatch;
+ } )
+ );
+
+ tunnel = await localTunnel( tunnelId );
+ if ( verbose ) {
+ console.log( "Started BrowserStackLocal." );
+ }
+ }
+
function queueRuns( suite, browser ) {
const fullBrowser = getBrowserString( browser, headless );
for ( const jquery of jquerys ) {
- const reportId = generateHash( `${ suite } ${ fullBrowser }` );
+ const reportId = generateHash( `${ suite } ${ jquery } ${ fullBrowser }` );
reports[ reportId ] = { browser, headless, jquery, migrate, suite };
const url = buildTestUrl( suite, {
+ browserstack,
jquery,
migrate,
port,
} );
const options = {
+ browserstack,
+ concurrency,
debug,
headless,
jquery,
migrate,
reportId,
+ runId,
suite,
+ tunnelId,
verbose
};
for ( const browser of browsers ) {
for ( const suite of suites ) {
- queueRuns( suite, browser );
+ queueRuns( [ suite ], browser );
}
}
try {
- await runAll( { concurrency, verbose } );
+ console.log( `Starting Run ${ runId }...` );
+ await runAll();
} catch ( error ) {
console.error( error );
if ( !debug ) {
}
console.log( chalk.green( "All tests passed!" ) );
- if ( !debug ) {
+ if ( !debug || browserstack ) {
gracefulExit( 0 );
}
} else {
if ( debug ) {
console.log();
- console.log( "Leaving browsers open for debugging." );
+ if ( browserstack ) {
+ console.log( "Leaving browsers with failures open for debugging." );
+ console.log(
+ "View running sessions at https://automate.browserstack.com/dashboard/v2/"
+ );
+ } else {
+ console.log( "Leaving browsers open for debugging." );
+ }
console.log( "Press Ctrl+C to exit." );
} else {
gracefulExit( 1 );
+++ /dev/null
-import chalk from "chalk";
-import { getBrowserString } from "../lib/getBrowserString.js";
-import createDriver from "./createDriver.js";
-
-const workers = Object.create( null );
-
-/**
- * Keys are browser strings
- * Structure of a worker:
- * {
- * debug: boolean, // Stops the worker from being cleaned up when finished
- * id: string,
- * lastTouch: number, // The last time a request was received
- * url: string,
- * browser: object, // The browser object
- * options: object // The options to create the worker
- * }
- */
-
-// Acknowledge the worker within the time limit.
-const ACKNOWLEDGE_INTERVAL = 1000;
-const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 1;
-
-const MAX_WORKER_RESTARTS = 5;
-
-// No report after the time limit
-// should refresh the worker
-const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
-
-const WORKER_WAIT_TIME = 30000;
-
-// Limit concurrency to 8 by default in selenium
-const MAX_CONCURRENCY = 8;
-
-export function touchBrowser( browser ) {
- const fullBrowser = getBrowserString( browser );
- const worker = workers[ fullBrowser ];
- if ( worker ) {
- worker.lastTouch = Date.now();
- }
-}
-
-async function waitForAck( worker, { fullBrowser, verbose } ) {
- delete worker.lastTouch;
- return new Promise( ( resolve, reject ) => {
- const interval = setInterval( () => {
- if ( worker.lastTouch ) {
- if ( verbose ) {
- console.log( `\n${ fullBrowser } acknowledged.` );
- }
- clearTimeout( timeout );
- clearInterval( interval );
- resolve();
- }
- }, ACKNOWLEDGE_INTERVAL );
-
- const timeout = setTimeout( () => {
- clearInterval( interval );
- reject(
- new Error(
- `${ fullBrowser } not acknowledged after ${
- ACKNOWLEDGE_TIMEOUT / 1000 / 60
- }min.`
- )
- );
- }, ACKNOWLEDGE_TIMEOUT );
- } );
-}
-
-async function restartWorker( worker ) {
- await cleanupWorker( worker, worker.options );
- await createBrowserWorker(
- worker.url,
- worker.browser,
- worker.options,
- worker.restarts + 1
- );
-}
-
-async function ensureAcknowledged( worker ) {
- const fullBrowser = getBrowserString( worker.browser );
- const verbose = worker.options.verbose;
- try {
- await waitForAck( worker, { fullBrowser, verbose } );
- return worker;
- } catch ( error ) {
- console.error( error.message );
- await restartWorker( worker );
- }
-}
-
-export async function createBrowserWorker( url, browser, options, restarts = 0 ) {
- if ( restarts > MAX_WORKER_RESTARTS ) {
- throw new Error(
- `Reached the maximum number of restarts for ${ chalk.yellow(
- getBrowserString( browser )
- ) }`
- );
- }
- const { concurrency = MAX_CONCURRENCY, debug, headless, verbose } = options;
- while ( workers.length >= concurrency ) {
- if ( verbose ) {
- console.log( "\nWaiting for available sessions..." );
- }
- await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
- }
-
- const fullBrowser = getBrowserString( browser );
-
- const driver = await createDriver( {
- browserName: browser.browser,
- headless,
- url,
- verbose
- } );
-
- const worker = {
- debug: !!debug,
- driver,
- url,
- browser,
- restarts,
- options
- };
-
- worker.debug = !!debug;
- worker.url = url;
- worker.browser = browser;
- worker.restarts = restarts;
- worker.options = options;
- touchBrowser( browser );
- workers[ fullBrowser ] = worker;
-
- // Wait for the worker to show up in the list
- // before returning it.
- return ensureAcknowledged( worker );
-}
-
-export async function setBrowserWorkerUrl( browser, url ) {
- const fullBrowser = getBrowserString( browser );
- const worker = workers[ fullBrowser ];
- if ( worker ) {
- worker.url = url;
- }
-}
-
-/**
- * Checks that all browsers have received
- * a response in the given amount of time.
- * If not, the worker is restarted.
- */
-export async function checkLastTouches() {
- for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) {
- if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) {
- const options = worker.options;
- if ( options.verbose ) {
- console.log(
- `\nNo response from ${ chalk.yellow( fullBrowser ) } in ${
- RUN_WORKER_TIMEOUT / 1000 / 60
- }min.`
- );
- }
- await restartWorker( worker );
- }
- }
-}
-
-export async function cleanupWorker( worker, { verbose } ) {
- for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
- if ( w === worker ) {
- delete workers[ fullBrowser ];
- await w.driver.quit();
- if ( verbose ) {
- console.log( `\nStopped ${ fullBrowser }.` );
- }
- return;
- }
- }
-}
-
-export async function cleanupAllBrowsers( { verbose } ) {
- const workersRemaining = Object.values( workers );
- const numRemaining = workersRemaining.length;
- if ( numRemaining ) {
- try {
- await Promise.all(
- workersRemaining.map( ( worker ) => worker.driver.quit() )
- );
- if ( verbose ) {
- console.log(
- `Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
- );
- }
- } catch ( error ) {
-
- // Log the error, but do not consider the test run failed
- console.error( error );
- }
- }
-}
+++ /dev/null
-import chalk from "chalk";
-import { getBrowserString } from "../lib/getBrowserString.js";
-import {
- checkLastTouches,
- createBrowserWorker,
- setBrowserWorkerUrl
-} from "./browsers.js";
-
-const TEST_POLL_TIMEOUT = 1000;
-
-const queue = [];
-
-export function getNextBrowserTest( reportId ) {
- const index = queue.findIndex( ( test ) => test.id === reportId );
- if ( index === -1 ) {
- return;
- }
-
- // Remove the completed test from the queue
- const previousTest = queue[ index ];
- queue.splice( index, 1 );
-
- // Find the next test for the same browser
- for ( const test of queue.slice( index ) ) {
- if ( test.fullBrowser === previousTest.fullBrowser ) {
-
- // Set the URL for our tracking
- setBrowserWorkerUrl( test.browser, test.url );
- test.running = true;
-
- // Return the URL for the next test.
- // listeners.js will use this to set the browser URL.
- return { url: test.url };
- }
- }
-}
-
-export function retryTest( reportId, maxRetries ) {
- if ( !maxRetries ) {
- return;
- }
- const test = queue.find( ( test ) => test.id === reportId );
- if ( test ) {
- test.retries++;
- if ( test.retries <= maxRetries ) {
- console.log(
- `\nRetrying test ${ reportId } for ${ chalk.yellow( test.options.suite ) }...${
- test.retries
- }`
- );
- return test;
- }
- }
-}
-
-export function addRun( url, browser, options ) {
- queue.push( {
- browser,
- fullBrowser: getBrowserString( browser ),
- id: options.reportId,
- retries: 0,
- url,
- options,
- running: false
- } );
-}
-
-export async function runAll() {
- return new Promise( async( resolve, reject ) => {
- while ( queue.length ) {
- try {
- await checkLastTouches();
- } catch ( error ) {
- reject( error );
- }
-
- // Run one test URL per browser at a time
- const browsersTaken = [];
- for ( const test of queue ) {
- if ( browsersTaken.indexOf( test.fullBrowser ) > -1 ) {
- continue;
- }
- browsersTaken.push( test.fullBrowser );
- if ( !test.running ) {
- test.running = true;
- try {
- await createBrowserWorker( test.url, test.browser, test.options );
- } catch ( error ) {
- reject( error );
- }
- }
- }
- await new Promise( ( resolve ) => setTimeout( resolve, TEST_POLL_TIMEOUT ) );
- }
- resolve();
- } );
-}
+++ /dev/null
-export const suites = [
- "accordion",
- "autocomplete",
- "button",
- "checkboxradio",
- "controlgroup",
- "core",
- "datepicker",
- "dialog",
- "draggable",
- "droppable",
- "effects",
- "form-reset-mixin",
- "menu",
- "position",
- "progressbar",
- "resizable",
- "selectable",
- "selectmenu",
- "slider",
- "sortable",
- "spinner",
- "tabs",
- "tooltip",
- "widget"
-];
projects / jquery-ui.git / commitdiff