From b6857536606d5d0dd1b07ada519f845cdac5c96e Mon Sep 17 00:00:00 2001 From: Timmy Willison Date: Tue, 14 Jan 2025 05:15:49 -0500 Subject: Tests: Migrate test runner to jquery-test-runner Closes gh-2325 --- .github/workflows/filestash.yml | 2 +- .github/workflows/node.js.yml | 42 ++- jtr-git.yml | 38 +++ jtr-stable.yml | 40 +++ jtr.yml | 30 ++ package.json | 17 +- tests/lib/qunit.js | 2 - tests/runner/.eslintrc.json | 38 --- tests/runner/browsers.js | 242 --------------- tests/runner/browserstack/api.js | 332 -------------------- .../runner/browserstack/buildBrowserFromString.js | 20 -- tests/runner/browserstack/createAuthHeader.js | 7 - tests/runner/browserstack/local.js | 34 --- tests/runner/command.js | 140 --------- tests/runner/createTestServer.js | 66 ---- tests/runner/flags/browsers.js | 24 -- tests/runner/flags/jquery.js | 14 - tests/runner/flags/suites.js | 27 -- tests/runner/lib/buildTestUrl.js | 24 -- tests/runner/lib/generateHash.js | 10 - tests/runner/lib/getBrowserString.js | 48 --- tests/runner/lib/prettyMs.js | 18 -- tests/runner/listeners.js | 112 ------- tests/runner/package.json | 3 - tests/runner/queue.js | 119 -------- tests/runner/reporter.js | 134 -------- tests/runner/run.js | 338 --------------------- tests/runner/selenium/createDriver.js | 84 ----- tests/runner/server.js | 13 - 29 files changed, 132 insertions(+), 1886 deletions(-) create mode 100644 jtr-git.yml create mode 100644 jtr-stable.yml create mode 100644 jtr.yml delete mode 100644 tests/runner/.eslintrc.json delete mode 100644 tests/runner/browsers.js delete mode 100644 tests/runner/browserstack/api.js delete mode 100644 tests/runner/browserstack/buildBrowserFromString.js delete mode 100644 tests/runner/browserstack/createAuthHeader.js delete mode 100644 tests/runner/browserstack/local.js delete mode 100644 tests/runner/command.js delete mode 100644 tests/runner/createTestServer.js delete mode 100644 tests/runner/flags/browsers.js delete mode 100644 tests/runner/flags/jquery.js delete mode 100644 tests/runner/flags/suites.js delete mode 100644 tests/runner/lib/buildTestUrl.js delete mode 100644 tests/runner/lib/generateHash.js delete mode 100644 tests/runner/lib/getBrowserString.js delete mode 100644 tests/runner/lib/prettyMs.js delete mode 100644 tests/runner/listeners.js delete mode 100644 tests/runner/package.json delete mode 100644 tests/runner/queue.js delete mode 100644 tests/runner/reporter.js delete mode 100644 tests/runner/run.js delete mode 100644 tests/runner/selenium/createDriver.js delete mode 100644 tests/runner/server.js diff --git a/.github/workflows/filestash.yml b/.github/workflows/filestash.yml index 8253df1ae..8de197a66 100644 --- a/.github/workflows/filestash.yml +++ b/.github/workflows/filestash.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest environment: filestash env: - NODE_VERSION: 20.x + NODE_VERSION: 22.x name: Update Filestash steps: - name: Checkout diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index bbabfb878..f23498ffd 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -12,21 +12,21 @@ permissions: contents: read env: - NODE_VERSION: 20.x + NODE_VERSION: 22.x jobs: build-and-test: runs-on: ubuntu-latest name: | - ${{ matrix.BROWSER }} | ${{ matrix.JQUERYS.name }} + ${{ matrix.BROWSER }} | ${{ matrix.CONFIGS.name }} strategy: fail-fast: false matrix: BROWSER: [chrome, firefox] - JQUERYS: - - versions: --jquery git --jquery 3.x-git + CONFIGS: + - config: jtr-git.yml name: jQuery git - - versions: --jquery 3.7.1 --jquery 3.6.4 --jquery 2.2.4 --jquery 1.12.4 + - config: jtr-stable.yml name: jQuery stable steps: @@ -57,21 +57,21 @@ jobs: - name: Test run: | - npm run test:unit -- -h -b ${{ matrix.BROWSER }} \ - ${{ matrix.JQUERYS.versions }} \ - --retries 3 --hard-retries 1 + npm run test:unit -- \ + --headless -b ${{ matrix.BROWSER }} \ + -c ${{ matrix.CONFIGS.config }} edge: runs-on: windows-latest name: | - edge | ${{ matrix.JQUERYS.name }} + edge | ${{ matrix.CONFIGS.name }} strategy: fail-fast: false matrix: - JQUERYS: - - versions: --jquery git --jquery 3.x-git + CONFIGS: + - config: jtr-git.yml name: jQuery git - - versions: --jquery 3.7.1 --jquery 3.6.4 --jquery 2.2.4 --jquery 1.12.4 + - config: jtr-stable.yml name: jQuery stable steps: - name: Checkout @@ -97,22 +97,19 @@ jobs: run: npm run build - name: Test - run: | - npm run test:unit -- -h -b edge ` - ${{ matrix.JQUERYS.versions }} ` - --retries 3 --hard-retries 1 + run: npm run test:unit -- -- --headless -b edge -c ${{ matrix.CONFIGS.config }} safari: runs-on: macos-latest name: | - safari | ${{ matrix.JQUERYS.name }} + safari | ${{ matrix.CONFIGS.name }} strategy: fail-fast: false matrix: - JQUERYS: - - versions: --jquery git --jquery 3.x-git + CONFIGS: + - config: jtr-git.yml name: jQuery git - - versions: --jquery 3.7.1 --jquery 3.6.4 --jquery 2.2.4 --jquery 1.12.4 + - config: jtr-stable.yml name: jQuery stable steps: - name: Checkout @@ -138,7 +135,4 @@ jobs: run: npm run build - name: Test - run: | - npm run test:unit -- -b safari \ - ${{ matrix.JQUERYS.versions }} \ - --retries 3 --hard-retries 1 + run: npm run test:unit -- -b safari -c ${{ matrix.CONFIGS.config }} diff --git a/jtr-git.yml b/jtr-git.yml new file mode 100644 index 000000000..0fc81ddd9 --- /dev/null +++ b/jtr-git.yml @@ -0,0 +1,38 @@ +version: 1 + +base-url: /tests/unit/ + +test-urls: + - accordion/accordion.html + - autocomplete/autocomplete.html + - button/button.html + - checkboxradio/checkboxradio.html + - controlgroup/controlgroup.html + - core/core.html + - datepicker/datepicker.html + - dialog/dialog.html + - draggable/draggable.html + - droppable/droppable.html + - effects/effects.html + - form-reset-mixin/form-reset-mixin.html + - jquery-patch/jquery-patch.html + - menu/menu.html + - position/position.html + - progressbar/progressbar.html + - resizable/resizable.html + - selectable/selectable.html + - selectmenu/selectmenu.html + - slider/slider.html + - sortable/sortable.html + - spinner/spinner.html + - tabs/tabs.html + - tooltip/tooltip.html + - widget/widget.html + +runs: + jquery: + - git + - 3.x-git + +retries: 2 +hard-retries: 1 diff --git a/jtr-stable.yml b/jtr-stable.yml new file mode 100644 index 000000000..1c6c27a10 --- /dev/null +++ b/jtr-stable.yml @@ -0,0 +1,40 @@ +version: 1 + +base-url: /tests/unit/ + +test-urls: + - accordion/accordion.html + - autocomplete/autocomplete.html + - button/button.html + - checkboxradio/checkboxradio.html + - controlgroup/controlgroup.html + - core/core.html + - datepicker/datepicker.html + - dialog/dialog.html + - draggable/draggable.html + - droppable/droppable.html + - effects/effects.html + - form-reset-mixin/form-reset-mixin.html + - jquery-patch/jquery-patch.html + - menu/menu.html + - position/position.html + - progressbar/progressbar.html + - resizable/resizable.html + - selectable/selectable.html + - selectmenu/selectmenu.html + - slider/slider.html + - sortable/sortable.html + - spinner/spinner.html + - tabs/tabs.html + - tooltip/tooltip.html + - widget/widget.html + +runs: + jquery: + - 3.7.1 + - 3.6.4 + - 2.2.4 + - 1.12.4 + +retries: 2 +hard-retries: 1 diff --git a/jtr.yml b/jtr.yml new file mode 100644 index 000000000..70d30ade8 --- /dev/null +++ b/jtr.yml @@ -0,0 +1,30 @@ +version: 1 + +base-url: /tests/unit/ + +test-urls: + - accordion/accordion.html + - autocomplete/autocomplete.html + - button/button.html + - checkboxradio/checkboxradio.html + - controlgroup/controlgroup.html + - core/core.html + - datepicker/datepicker.html + - dialog/dialog.html + - draggable/draggable.html + - droppable/droppable.html + - effects/effects.html + - form-reset-mixin/form-reset-mixin.html + - jquery-patch/jquery-patch.html + - menu/menu.html + - position/position.html + - progressbar/progressbar.html + - resizable/resizable.html + - selectable/selectable.html + - selectmenu/selectmenu.html + - slider/slider.html + - sortable/sortable.html + - spinner/spinner.html + - tabs/tabs.html + - tooltip/tooltip.html + - widget/widget.html diff --git a/package.json b/package.json index f82758844..56f60f34d 100644 --- a/package.json +++ b/package.json @@ -47,22 +47,16 @@ "scripts": { "build": "grunt build", "lint": "grunt lint", - "test:server": "node tests/runner/server.js", - "test:unit": "node tests/runner/command.js", - "test": "grunt && npm run test:unit -- -h" + "test:server": "jtr serve", + "test:unit": "jtr", + "test": "grunt && npm run test:unit -- --headless" }, "dependencies": { "jquery": ">=1.12.0 <5.0.0" }, "devDependencies": { - "body-parser": "1.20.3", - "browserstack-local": "1.5.5", "commitplease": "3.2.0", - "diff": "5.2.0", "eslint-config-jquery": "3.0.2", - "exit-hook": "4.0.0", - "express": "4.21.1", - "express-body-parser-error-handler": "1.0.7", "grunt": "1.6.1", "grunt-bowercopy": "1.2.5", "grunt-compare-size": "0.4.2", @@ -73,10 +67,9 @@ "grunt-eslint": "24.0.1", "grunt-git-authors": "3.2.0", "grunt-html": "17.1.0", + "jquery-test-runner": "0.2.1", "load-grunt-tasks": "5.1.0", - "rimraf": "6.0.1", - "selenium-webdriver": "4.26.0", - "yargs": "17.7.2" + "rimraf": "6.0.1" }, "keywords": [] } diff --git a/tests/lib/qunit.js b/tests/lib/qunit.js index cc2f01d79..6441019bd 100644 --- a/tests/lib/qunit.js +++ b/tests/lib/qunit.js @@ -13,8 +13,6 @@ QUnit.config.requireExpects = true; QUnit.config.urlConfig.push( { id: "jquery", label: "jQuery version", - - // Keep in sync with tests/runner/jquery.js value: [ "1.12.4", "2.2.4", diff --git a/tests/runner/.eslintrc.json b/tests/runner/.eslintrc.json deleted file mode 100644 index 9ca2e75f6..000000000 --- a/tests/runner/.eslintrc.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "root": true, - - "extends": "jquery", - - "overrides": [ - { - "files": ["**/*"], - "env": { - "es6": true, - "node": true - }, - "parserOptions": { - "ecmaVersion": 2022, - "sourceType": "module" - } - }, - { - "files": ["./listeners.js"], - "env": { - "browser": true, - "node": false - }, - "globals": { - "QUnit": false, - "Symbol": false, - "require": false - }, - "parserOptions": { - "ecmaVersion": 5, - "sourceType": "script" - }, - "rules": { - "strict": ["error", "function"] - } - } - ] -} diff --git a/tests/runner/browsers.js b/tests/runner/browsers.js deleted file mode 100644 index 1ddccdf78..000000000 --- a/tests/runner/browsers.js +++ /dev/null @@ -1,242 +0,0 @@ -import chalk from "chalk"; -import { getBrowserString } from "./lib/getBrowserString.js"; -import { - createWorker, - deleteWorker, - getAvailableSessions -} from "./browserstack/api.js"; -import createDriver from "./selenium/createDriver.js"; - -const workers = Object.create( null ); - -/** - * Keys are browser strings - * Structure of a worker: - * { - * browser: object // The browser object - * debug: boolean // Stops the worker from being cleaned up when finished - * lastTouch: number // The last time a request was received - * restarts: number // The number of times the worker has been restarted - * options: object // The options to create the worker - * url: string // The URL the worker is on - * quit: function // A function to stop the worker - * } - */ - -// Acknowledge the worker within the time limit. -// BrowserStack can take much longer spinning up -// some browsers, such as iOS 15 Safari. -const ACKNOWLEDGE_INTERVAL = 1000; -const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 5; - -const MAX_WORKER_RESTARTS = 5; - -// No report after the time limit -// should refresh the worker -const RUN_WORKER_TIMEOUT = 60 * 1000 * 2; - -const WORKER_WAIT_TIME = 30000; - -// Limit concurrency to 8 by default in selenium -const MAX_SELENIUM_CONCURRENCY = 8; - -export async function createBrowserWorker( url, browser, options, restarts = 0 ) { - if ( restarts > MAX_WORKER_RESTARTS ) { - throw new Error( - `Reached the maximum number of restarts for ${ chalk.yellow( - getBrowserString( browser ) - ) }` - ); - } - const { browserstack, debug, headless, runId, tunnelId, verbose } = options; - while ( await maxWorkersReached( options ) ) { - if ( verbose ) { - console.log( "\nWaiting for available sessions..." ); - } - await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) ); - } - - const fullBrowser = getBrowserString( browser ); - - let worker; - - if ( browserstack ) { - worker = await createWorker( { - ...browser, - url: encodeURI( url ), - project: "jquery", - build: `Run ${ runId }`, - - // This is the maximum timeout allowed - // by BrowserStack. We do this because - // we control the timeout in the runner. - // See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300 - timeout: 1800, - - // Not documented in the API docs, - // but required to make local testing work. - // See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs - "browserstack.local": true, - "browserstack.localIdentifier": tunnelId - } ); - worker.quit = () => deleteWorker( worker.id ); - } else { - const driver = await createDriver( { - browserName: browser.browser, - headless, - url, - verbose - } ); - worker = { - quit: () => driver.quit() - }; - } - - worker.debug = !!debug; - worker.url = url; - worker.browser = browser; - worker.restarts = restarts; - worker.options = options; - touchBrowser( browser ); - workers[ fullBrowser ] = worker; - - // Wait for the worker to show up in the list - // before returning it. - return ensureAcknowledged( worker ); -} - -export function touchBrowser( browser ) { - const fullBrowser = getBrowserString( browser ); - const worker = workers[ fullBrowser ]; - if ( worker ) { - worker.lastTouch = Date.now(); - } -} - -export async function setBrowserWorkerUrl( browser, url ) { - const fullBrowser = getBrowserString( browser ); - const worker = workers[ fullBrowser ]; - if ( worker ) { - worker.url = url; - } -} - -export async function restartBrowser( browser ) { - const fullBrowser = getBrowserString( browser ); - const worker = workers[ fullBrowser ]; - if ( worker ) { - await restartWorker( worker ); - } -} - -/** - * Checks that all browsers have received - * a response in the given amount of time. - * If not, the worker is restarted. - */ -export async function checkLastTouches() { - for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) { - if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) { - const options = worker.options; - if ( options.verbose ) { - console.log( - `\nNo response from ${ chalk.yellow( fullBrowser ) } in ${ - RUN_WORKER_TIMEOUT / 1000 / 60 - }min.` - ); - } - await restartWorker( worker ); - } - } -} - -export async function cleanupAllBrowsers( { verbose } ) { - const workersRemaining = Object.values( workers ); - const numRemaining = workersRemaining.length; - if ( numRemaining ) { - try { - await Promise.all( workersRemaining.map( ( worker ) => worker.quit() ) ); - if ( verbose ) { - console.log( - `Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.` - ); - } - } catch ( error ) { - - // Log the error, but do not consider the test run failed - console.error( error ); - } - } -} - -async function maxWorkersReached( { - browserstack, - concurrency = MAX_SELENIUM_CONCURRENCY -} ) { - if ( browserstack ) { - return ( await getAvailableSessions() ) <= 0; - } - return workers.length >= concurrency; -} - -async function waitForAck( worker, { fullBrowser, verbose } ) { - delete worker.lastTouch; - return new Promise( ( resolve, reject ) => { - const interval = setInterval( () => { - if ( worker.lastTouch ) { - if ( verbose ) { - console.log( `\n${ fullBrowser } acknowledged.` ); - } - clearTimeout( timeout ); - clearInterval( interval ); - resolve(); - } - }, ACKNOWLEDGE_INTERVAL ); - - const timeout = setTimeout( () => { - clearInterval( interval ); - reject( - new Error( - `${ fullBrowser } not acknowledged after ${ - ACKNOWLEDGE_TIMEOUT / 1000 / 60 - }min.` - ) - ); - }, ACKNOWLEDGE_TIMEOUT ); - } ); -} - -async function ensureAcknowledged( worker ) { - const fullBrowser = getBrowserString( worker.browser ); - const verbose = worker.options.verbose; - try { - await waitForAck( worker, { fullBrowser, verbose } ); - return worker; - } catch ( error ) { - console.error( error.message ); - await restartWorker( worker ); - } -} - -async function cleanupWorker( worker, { verbose } ) { - for ( const [ fullBrowser, w ] of Object.entries( workers ) ) { - if ( w === worker ) { - delete workers[ fullBrowser ]; - await worker.quit(); - if ( verbose ) { - console.log( `\nStopped ${ fullBrowser }.` ); - } - return; - } - } -} - -async function restartWorker( worker ) { - await cleanupWorker( worker, worker.options ); - await createBrowserWorker( - worker.url, - worker.browser, - worker.options, - worker.restarts + 1 - ); -} diff --git a/tests/runner/browserstack/api.js b/tests/runner/browserstack/api.js deleted file mode 100644 index 632f90c3b..000000000 --- a/tests/runner/browserstack/api.js +++ /dev/null @@ -1,332 +0,0 @@ -/** - * Browserstack API is documented at - * https://github.com/browserstack/api - */ - -import { createAuthHeader } from "./createAuthHeader.js"; - -const browserstackApi = "https://api.browserstack.com"; -const apiVersion = 5; - -const username = process.env.BROWSERSTACK_USERNAME; -const accessKey = process.env.BROWSERSTACK_ACCESS_KEY; - -// iOS has null for version numbers, -// and we do not need a similar check for OS versions. -const rfinalVersion = /(?:^[0-9\.]+$)|(?:^null$)/; -const rlatest = /^latest-(\d+)$/; - -const rnonDigits = /(?:[^\d\.]+)|(?:20\d{2})/g; - -async function fetchAPI( path, options = {}, versioned = true ) { - if ( !username || !accessKey ) { - throw new Error( - "BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables must be set." - ); - } - const init = { - method: "GET", - ...options, - headers: { - authorization: createAuthHeader( username, accessKey ), - accept: "application/json", - "content-type": "application/json", - ...options.headers - } - }; - const response = await fetch( - `${ browserstackApi }/${ versioned ? `${ apiVersion }/` : "" }${ path }`, - init - ); - if ( !response.ok ) { - console.log( - `\n${ init.method } ${ path }`, - response.status, - response.statusText - ); - throw new Error( `Error fetching ${ path }` ); - } - return response.json(); -} - -/** - * ============================= - * Browsers API - * ============================= - */ - -function compareVersionNumbers( a, b ) { - if ( a != null && b == null ) { - return -1; - } - if ( a == null && b != null ) { - return 1; - } - if ( a == null && b == null ) { - return 0; - } - const aParts = a.replace( rnonDigits, "" ).split( "." ); - const bParts = b.replace( rnonDigits, "" ).split( "." ); - - if ( aParts.length > bParts.length ) { - return -1; - } - if ( aParts.length < bParts.length ) { - return 1; - } - - for ( let i = 0; i < aParts.length; i++ ) { - const aPart = Number( aParts[ i ] ); - const bPart = Number( bParts[ i ] ); - if ( aPart < bPart ) { - return -1; - } - if ( aPart > bPart ) { - return 1; - } - } - - if ( rnonDigits.test( a ) && !rnonDigits.test( b ) ) { - return -1; - } - if ( !rnonDigits.test( a ) && rnonDigits.test( b ) ) { - return 1; - } - - return 0; -} - -function sortBrowsers( a, b ) { - if ( a.browser < b.browser ) { - return -1; - } - if ( a.browser > b.browser ) { - return 1; - } - const browserComparison = compareVersionNumbers( - a.browser_version, - b.browser_version - ); - if ( browserComparison ) { - return browserComparison; - } - if ( a.os < b.os ) { - return -1; - } - if ( a.os > b.os ) { - return 1; - } - const osComparison = compareVersionNumbers( a.os_version, b.os_version ); - if ( osComparison ) { - return osComparison; - } - const deviceComparison = compareVersionNumbers( a.device, b.device ); - if ( deviceComparison ) { - return deviceComparison; - } - return 0; -} - -export async function getBrowsers( { flat = false } = {} ) { - const query = new URLSearchParams(); - if ( flat ) { - query.append( "flat", true ); - } - const browsers = await fetchAPI( `/browsers?${ query }` ); - return browsers.sort( sortBrowsers ); -} - -function matchVersion( browserVersion, version ) { - if ( !version ) { - return false; - } - const regex = new RegExp( - `^${ version.replace( /\\/g, "\\\\" ).replace( /\./g, "\\." ) }\\b`, - "i" - ); - return regex.test( browserVersion ); -} - -export async function filterBrowsers( filter ) { - const browsers = await getBrowsers( { flat: true } ); - if ( !filter ) { - return browsers; - } - const filterBrowser = ( filter.browser ?? "" ).toLowerCase(); - const filterVersion = ( filter.browser_version ?? "" ).toLowerCase(); - const filterOs = ( filter.os ?? "" ).toLowerCase(); - const filterOsVersion = ( filter.os_version ?? "" ).toLowerCase(); - const filterDevice = ( filter.device ?? "" ).toLowerCase(); - - const filteredWithoutVersion = browsers.filter( ( browser ) => { - return ( - ( !filterBrowser || filterBrowser === browser.browser.toLowerCase() ) && - ( !filterOs || filterOs === browser.os.toLowerCase() ) && - ( !filterOsVersion || matchVersion( browser.os_version, filterOsVersion ) ) && - ( !filterDevice || filterDevice === ( browser.device || "" ).toLowerCase() ) - ); - } ); - - if ( !filterVersion ) { - return filteredWithoutVersion; - } - - if ( filterVersion.startsWith( "latest" ) ) { - const groupedByName = filteredWithoutVersion - .filter( ( b ) => rfinalVersion.test( b.browser_version ) ) - .reduce( ( acc, browser ) => { - acc[ browser.browser ] = acc[ browser.browser ] ?? []; - acc[ browser.browser ].push( browser ); - return acc; - }, Object.create( null ) ); - - const filtered = []; - for ( const group of Object.values( groupedByName ) ) { - const latest = group[ group.length - 1 ]; - - // Mobile devices do not have browser version. - // Skip the version check for these, - // but include the latest in the list if it made it - // through filtering. - if ( !latest.browser_version ) { - - // Do not include in the list for latest-n. - if ( filterVersion === "latest" ) { - filtered.push( latest ); - } - continue; - } - - // Get the latest version and subtract the number from the filter, - // ignoring any patch versions, which may differ between major versions. - const num = rlatest.exec( filterVersion ); - const version = parseInt( latest.browser_version ) - ( num ? num[ 1 ] : 0 ); - const match = group.findLast( ( browser ) => { - return matchVersion( browser.browser_version, version.toString() ); - } ); - if ( match ) { - filtered.push( match ); - } - } - return filtered; - } - - return filteredWithoutVersion.filter( ( browser ) => { - return matchVersion( browser.browser_version, filterVersion ); - } ); -} - -export async function listBrowsers( filter ) { - const browsers = await filterBrowsers( filter ); - console.log( "Available browsers:" ); - for ( const browser of browsers ) { - let message = ` ${ browser.browser }_`; - if ( browser.device ) { - message += `:${ browser.device }_`; - } else { - message += `${ browser.browser_version }_`; - } - message += `${ browser.os }_${ browser.os_version }`; - console.log( message ); - } -} - -export async function getLatestBrowser( filter ) { - if ( !filter.browser_version ) { - filter.browser_version = "latest"; - } - const browsers = await filterBrowsers( filter ); - return browsers[ browsers.length - 1 ]; -} - -/** - * ============================= - * Workers API - * ============================= - */ - -/** - * A browser object may only have one of `browser` or `device` set; - * which property is set will depend on `os`. - * - * `options`: is an object with the following properties: - * `os`: The operating system. - * `os_version`: The operating system version. - * `browser`: The browser name. - * `browser_version`: The browser version. - * `device`: The device name. - * `url` (optional): Which URL to navigate to upon creation. - * `timeout` (optional): Maximum life of the worker (in seconds). Maximum value of `1800`. Specifying `0` will use the default of `300`. - * `name` (optional): Provide a name for the worker. - * `build` (optional): Group workers into a build. - * `project` (optional): Provide the project the worker belongs to. - * `resolution` (optional): Specify the screen resolution (e.g. "1024x768"). - * `browserstack.local` (optional): Set to `true` to mark as local testing. - * `browserstack.video` (optional): Set to `false` to disable video recording. - * `browserstack.localIdentifier` (optional): ID of the local tunnel. - */ -export function createWorker( options ) { - return fetchAPI( "/worker", { - method: "POST", - body: JSON.stringify( options ) - } ); -} - -/** - * Returns a worker object, if one exists, with the following properties: - * `id`: The worker id. - * `status`: A string representing the current status of the worker. - * Possible statuses: `"running"`, `"queue"`. - */ -export function getWorker( id ) { - return fetchAPI( `/worker/${ id }` ); -} - -export async function deleteWorker( id ) { - return fetchAPI( `/worker/${ id }`, { method: "DELETE" } ); -} - -export function getWorkers() { - return fetchAPI( "/workers" ); -} - -/** - * Stop all workers - */ -export async function stopWorkers() { - const workers = await getWorkers(); - - // Run each request on its own - // to avoid connect timeout errors. - console.log( `${ workers.length } workers running...` ); - for ( const worker of workers ) { - try { - await deleteWorker( worker.id ); - } catch ( error ) { - - // Log the error, but continue trying to remove workers. - console.error( error ); - } - } - console.log( "All workers stopped." ); -} - -/** - * ============================= - * Plan API - * ============================= - */ - -export function getPlan() { - return fetchAPI( "/automate/plan.json", {}, false ); -} - -export async function getAvailableSessions() { - try { - const [ plan, workers ] = await Promise.all( [ getPlan(), getWorkers() ] ); - return plan.parallel_sessions_max_allowed - workers.length; - } catch ( error ) { - console.error( error ); - return 0; - } -} diff --git a/tests/runner/browserstack/buildBrowserFromString.js b/tests/runner/browserstack/buildBrowserFromString.js deleted file mode 100644 index e0d99a039..000000000 --- a/tests/runner/browserstack/buildBrowserFromString.js +++ /dev/null @@ -1,20 +0,0 @@ -export function buildBrowserFromString( str ) { - const [ browser, versionOrDevice, os, osVersion ] = str.split( "_" ); - - // If the version starts with a colon, it's a device - if ( versionOrDevice && versionOrDevice.startsWith( ":" ) ) { - return { - browser, - device: versionOrDevice.slice( 1 ), - os, - os_version: osVersion - }; - } - - return { - browser, - browser_version: versionOrDevice, - os, - os_version: osVersion - }; -} diff --git a/tests/runner/browserstack/createAuthHeader.js b/tests/runner/browserstack/createAuthHeader.js deleted file mode 100644 index fe4831e9a..000000000 --- a/tests/runner/browserstack/createAuthHeader.js +++ /dev/null @@ -1,7 +0,0 @@ -const textEncoder = new TextEncoder(); - -export function createAuthHeader( username, accessKey ) { - const encoded = textEncoder.encode( `${ username }:${ accessKey }` ); - const base64 = btoa( String.fromCodePoint.apply( null, encoded ) ); - return `Basic ${ base64 }`; -} diff --git a/tests/runner/browserstack/local.js b/tests/runner/browserstack/local.js deleted file mode 100644 index c84cf155c..000000000 --- a/tests/runner/browserstack/local.js +++ /dev/null @@ -1,34 +0,0 @@ -import browserstackLocal from "browserstack-local"; - -export async function localTunnel( localIdentifier, opts = {} ) { - const tunnel = new browserstackLocal.Local(); - - return new Promise( ( resolve, reject ) => { - - // https://www.browserstack.com/docs/local-testing/binary-params - tunnel.start( - { - "enable-logging-for-api": "", - localIdentifier, - ...opts - }, - async( error ) => { - if ( error || !tunnel.isRunning() ) { - return reject( error ); - } - resolve( { - stop: function stopTunnel() { - return new Promise( ( resolve, reject ) => { - tunnel.stop( ( error ) => { - if ( error ) { - return reject( error ); - } - resolve(); - } ); - } ); - } - } ); - } - ); - } ); -} diff --git a/tests/runner/command.js b/tests/runner/command.js deleted file mode 100644 index cf5ddd8ee..000000000 --- a/tests/runner/command.js +++ /dev/null @@ -1,140 +0,0 @@ -import yargs from "yargs/yargs"; -import { browsers } from "./flags/browsers.js"; -import { getPlan, listBrowsers, stopWorkers } from "./browserstack/api.js"; -import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js"; -import { jquery } from "./flags/jquery.js"; -import { suites } from "./flags/suites.js"; -import { run } from "./run.js"; - -const argv = yargs( process.argv.slice( 2 ) ) - .version( false ) - .strict() - .command( { - command: "[options]", - describe: "Run jQuery tests in a browser" - } ) - .option( "suite", { - alias: "s", - type: "array", - choices: suites, - description: - "Run tests for a specific test suite.\n" + - "Pass multiple test suites by repeating the option.\n" + - "Defaults to all suites." - } ) - .option( "jquery", { - alias: "j", - type: "array", - choices: jquery, - description: - "Run tests against a specific jQuery version.\n" + - "Pass multiple versions by repeating the option.", - default: [ "3.7.1" ] - } ) - .option( "migrate", { - type: "boolean", - description: - "Run tests with jQuery Migrate enabled.", - default: false - } ) - .option( "browser", { - alias: "b", - type: "array", - choices: browsers, - description: - "Run tests in a specific browser." + - "Pass multiple browsers by repeating the option." + - "If using BrowserStack, specify browsers using --browserstack.", - default: [ "chrome" ] - } ) - .option( "headless", { - alias: "h", - type: "boolean", - description: - "Run tests in headless mode. Cannot be used with --debug or --browserstack.", - conflicts: [ "debug", "browserstack" ] - } ) - .option( "concurrency", { - alias: "c", - type: "number", - description: - "Run tests in parallel in multiple browsers. " + - "Defaults to 8 in normal mode. In browserstack mode, " + - "defaults to the maximum available under your BrowserStack plan." - } ) - .option( "debug", { - alias: "d", - type: "boolean", - description: - "Leave the browser open for debugging. Cannot be used with --headless.", - conflicts: [ "headless" ] - } ) - .option( "retries", { - alias: "r", - type: "number", - description: "Number of times to retry failed tests by refreshing the URL." - } ) - .option( "hard-retries", { - type: "number", - description: - "Number of times to retry failed tests by restarting the worker. " + - "This is in addition to the normal retries " + - "and are only used when the normal retries are exhausted." - } ) - .option( "verbose", { - alias: "v", - type: "boolean", - description: "Log additional information." - } ) - .option( "browserstack", { - type: "array", - description: - "Run tests in BrowserStack.\n" + - "Requires BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables.\n" + - "The value can be empty for the default configuration, or a string in the format of\n" + - "\"browser_[browserVersion | :device]_os_osVersion\" (see --list-browsers).\n" + - "Pass multiple browsers by repeating the option.\n" + - "The --browser option is ignored when --browserstack has a value.\n" + - "Otherwise, the --browser option will be used, " + - "with the latest version/device for that browser, on a matching OS." - } ) - .option( "run-id", { - type: "string", - description: "A unique identifier for the run in BrowserStack." - } ) - .option( "list-browsers", { - type: "string", - description: - "List available BrowserStack browsers and exit.\n" + - "Leave blank to view all browsers or pass " + - "\"browser_[browserVersion | :device]_os_osVersion\" with each parameter " + - "separated by an underscore to filter the list (any can be omitted).\n" + - "\"latest\" can be used in place of \"browserVersion\" to find the latest version.\n" + - "\"latest-n\" can be used to find the nth latest browser version.\n" + - "Use a colon to indicate a device.\n" + - "Examples: \"chrome__windows_10\", \"safari_latest\", " + - "\"Mobile Safari\", \"Android Browser_:Google Pixel 8 Pro\".\n" + - "Use quotes if spaces are necessary." - } ) - .option( "stop-workers", { - type: "boolean", - description: - "WARNING: This will stop all BrowserStack workers that may exist and exit," + - "including any workers running from other projects.\n" + - "This can be used as a failsafe when there are too many stray workers." - } ) - .option( "browserstack-plan", { - type: "boolean", - description: "Show BrowserStack plan information and exit." - } ) - .help().argv; - -if ( typeof argv.listBrowsers === "string" ) { - listBrowsers( buildBrowserFromString( argv.listBrowsers ) ); -} else if ( argv.stopWorkers ) { - stopWorkers(); -} else if ( argv.browserstackPlan ) { - console.log( await getPlan() ); -} else { - run( argv ); -} diff --git a/tests/runner/createTestServer.js b/tests/runner/createTestServer.js deleted file mode 100644 index 875e6d3b1..000000000 --- a/tests/runner/createTestServer.js +++ /dev/null @@ -1,66 +0,0 @@ -import { readFile } from "node:fs/promises"; -import bodyParser from "body-parser"; -import express from "express"; -import bodyParserErrorHandler from "express-body-parser-error-handler"; - -export async function createTestServer( report ) { - const app = express(); - - // Redirect home to test page - app.get( "/", ( _req, res ) => { - res.redirect( "/tests/" ); - } ); - - // Redirect to trailing slash - app.use( ( req, res, next ) => { - if ( req.path === "/tests" ) { - const query = req.url.slice( req.path.length ); - res.redirect( 301, `${ req.path }/${ query }` ); - } else { - next(); - } - } ); - - // Add a script tag to HTML pages to load the QUnit listeners - app.use( /\/tests\/unit\/([a-zA-Z0-9_-]+)\/\1\.html$/, async( req, res ) => { - const html = await readFile( - `tests/unit/${ req.params[ 0 ] }/${ req.params[ 0 ] }.html`, - "utf8" - ); - res.send( - html.replace( - "", - "" - ) - ); - } ); - - // Bind the reporter - app.post( - "/api/report", - bodyParser.json( { limit: "50mb" } ), - async( req, res ) => { - if ( report ) { - const response = await report( req.body ); - if ( response ) { - res.json( response ); - return; - } - } - res.sendStatus( 204 ); - } - ); - - // Handle errors from the body parser - app.use( bodyParserErrorHandler() ); - - // Serve static files - app.use( "/dist", express.static( "dist" ) ); - app.use( "/src", express.static( "src" ) ); - app.use( "/tests", express.static( "tests" ) ); - app.use( "/ui", express.static( "ui" ) ); - app.use( "/themes", express.static( "themes" ) ); - app.use( "/external", express.static( "external" ) ); - - return app; -} diff --git a/tests/runner/flags/browsers.js b/tests/runner/flags/browsers.js deleted file mode 100644 index 5d2306afe..000000000 --- a/tests/runner/flags/browsers.js +++ /dev/null @@ -1,24 +0,0 @@ -// This list is static, so no requests are required -// in the command help menu. - -import { getBrowsers } from "../browserstack/api.js"; - -export const browsers = [ - "chrome", - "ie", - "firefox", - "edge", - "safari", - "opera", - "yandex", - "IE Mobile", - "Android Browser", - "Mobile Safari" -]; - -// A function that can be used to update the above list. -export async function getAvailableBrowsers() { - const browsers = await getBrowsers( { flat: true } ); - const available = [ ...new Set( browsers.map( ( { browser } ) => browser ) ) ]; - return available; -} diff --git a/tests/runner/flags/jquery.js b/tests/runner/flags/jquery.js deleted file mode 100644 index 0d4f21524..000000000 --- a/tests/runner/flags/jquery.js +++ /dev/null @@ -1,14 +0,0 @@ -// Keep in sync with tests/lib/qunit.js -export const jquery = [ - "1.12.4", - "2.2.4", - "3.0.0", - "3.1.0", "3.1.1", - "3.2.0", "3.2.1", - "3.3.0", "3.3.1", - "3.4.0", "3.4.1", - "3.5.0", "3.5.1", - "3.6.0", "3.6.1", "3.6.2", "3.6.3", "3.6.4", - "3.7.0", "3.7.1", - "3.x-git", "git", "custom" -]; diff --git a/tests/runner/flags/suites.js b/tests/runner/flags/suites.js deleted file mode 100644 index a635ac4e5..000000000 --- a/tests/runner/flags/suites.js +++ /dev/null @@ -1,27 +0,0 @@ -export const suites = [ - "accordion", - "autocomplete", - "button", - "checkboxradio", - "controlgroup", - "core", - "datepicker", - "dialog", - "draggable", - "droppable", - "effects", - "form-reset-mixin", - "jquery-patch", - "menu", - "position", - "progressbar", - "resizable", - "selectable", - "selectmenu", - "slider", - "sortable", - "spinner", - "tabs", - "tooltip", - "widget" -]; diff --git a/tests/runner/lib/buildTestUrl.js b/tests/runner/lib/buildTestUrl.js deleted file mode 100644 index 5eb3b049b..000000000 --- a/tests/runner/lib/buildTestUrl.js +++ /dev/null @@ -1,24 +0,0 @@ -export function buildTestUrl( suite, { browserstack, jquery, migrate, port, reportId } ) { - if ( !port ) { - throw new Error( "No port specified." ); - } - - const query = new URLSearchParams(); - - if ( jquery ) { - query.append( "jquery", jquery ); - } - - if ( migrate ) { - query.append( "migrate", "true" ); - } - - if ( reportId ) { - query.append( "reportId", reportId ); - } - - // BrowserStack supplies a custom domain for local testing, - // which is especially necessary for iOS testing. - const host = browserstack ? "bs-local.com" : "localhost"; - return `http://${ host }:${ port }/tests/unit/${ suite }/${ suite }.html?${ query }`; -} diff --git a/tests/runner/lib/generateHash.js b/tests/runner/lib/generateHash.js deleted file mode 100644 index 66f2161d5..000000000 --- a/tests/runner/lib/generateHash.js +++ /dev/null @@ -1,10 +0,0 @@ -import crypto from "node:crypto"; - -export function generateHash( string ) { - const hash = crypto.createHash( "md5" ); - hash.update( string ); - - // QUnit hashes are 8 characters long - // We use 10 characters to be more visually distinct - return hash.digest( "hex" ).slice( 0, 10 ); -} diff --git a/tests/runner/lib/getBrowserString.js b/tests/runner/lib/getBrowserString.js deleted file mode 100644 index 0d293074c..000000000 --- a/tests/runner/lib/getBrowserString.js +++ /dev/null @@ -1,48 +0,0 @@ -const browserMap = { - chrome: "Chrome", - edge: "Edge", - firefox: "Firefox", - ie: "IE", - opera: "Opera", - safari: "Safari" -}; - -export function browserSupportsHeadless( browser ) { - browser = browser.toLowerCase(); - return ( - browser === "chrome" || - browser === "firefox" || - browser === "edge" - ); -} - -export function getBrowserString( - { - browser, - browser_version: browserVersion, - device, - os, - os_version: osVersion - }, - headless -) { - browser = browser.toLowerCase(); - browser = browserMap[ browser ] || browser; - let str = browser; - if ( browserVersion ) { - str += ` ${ browserVersion }`; - } - if ( device ) { - str += ` for ${ device }`; - } - if ( os ) { - str += ` on ${ os }`; - } - if ( osVersion ) { - str += ` ${ osVersion }`; - } - if ( headless && browserSupportsHeadless( browser ) ) { - str += " (headless)"; - } - return str; -} diff --git a/tests/runner/lib/prettyMs.js b/tests/runner/lib/prettyMs.js deleted file mode 100644 index 99bae2b35..000000000 --- a/tests/runner/lib/prettyMs.js +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Pretty print a time in milliseconds. - */ -export function prettyMs( time ) { - const minutes = Math.floor( time / 60000 ); - const seconds = Math.floor( time / 1000 ); - const ms = Math.floor( time % 1000 ); - - let prettyTime = `${ ms }ms`; - if ( seconds > 0 ) { - prettyTime = `${ seconds }s ${ prettyTime }`; - } - if ( minutes > 0 ) { - prettyTime = `${ minutes }m ${ prettyTime }`; - } - - return prettyTime; -} diff --git a/tests/runner/listeners.js b/tests/runner/listeners.js deleted file mode 100644 index ed6fb24e8..000000000 --- a/tests/runner/listeners.js +++ /dev/null @@ -1,112 +0,0 @@ -( function() { - "use strict"; - - // Get the report ID from the URL. - var match = location.search.match( /reportId=([^&]+)/ ); - if ( !match ) { - return; - } - var id = match[ 1 ]; - - // Adopted from https://github.com/douglascrockford/JSON-js - // Support: IE 11+ - // Using the replacer argument of JSON.stringify in IE has issues - // TODO: Replace this with a circular replacer + JSON.stringify + WeakSet - function decycle( object ) { - var objects = []; - - // The derez function recurses through the object, producing the deep copy. - function derez( value ) { - if ( - typeof value === "object" && - value !== null && - !( value instanceof Boolean ) && - !( value instanceof Date ) && - !( value instanceof Number ) && - !( value instanceof RegExp ) && - !( value instanceof String ) - ) { - - // Return a string early for elements - if ( value.nodeType ) { - return value.toString(); - } - - if ( objects.indexOf( value ) > -1 ) { - return; - } - - objects.push( value ); - - if ( Array.isArray( value ) ) { - - // If it is an array, replicate the array. - return value.map( derez ); - } else { - - // If it is an object, replicate the object. - var nu = Object.create( null ); - Object.keys( value ).forEach( function( name ) { - nu[ name ] = derez( value[ name ] ); - } ); - return nu; - } - } - - // Serialize Symbols as string representations so they are - // sent over the wire after being stringified. - if ( typeof value === "symbol" ) { - - // We can *describe* unique symbols, but note that their identity - // (e.g., `Symbol() !== Symbol()`) is lost - var ctor = Symbol.keyFor( value ) !== undefined ? "Symbol.for" : "Symbol"; - return ctor + "(" + JSON.stringify( value.description ) + ")"; - } - - return value; - } - return derez( object ); - } - - function send( type, data ) { - var json = JSON.stringify( { - id: id, - type: type, - data: data ? decycle( data ) : undefined - } ); - var request = new XMLHttpRequest(); - request.open( "POST", "/api/report", true ); - request.setRequestHeader( "Content-Type", "application/json" ); - request.send( json ); - return request; - } - - require( [ "qunit" ], function( QUnit ) { - - // Send acknowledgement to the server. - send( "ack" ); - - QUnit.on( "testEnd", function( data ) { - send( "testEnd", data ); - } ); - - QUnit.on( "runEnd", function( data ) { - - // Reduce the payload size. - // childSuites is large and unused. - data.childSuites = undefined; - - var request = send( "runEnd", data ); - request.onload = function() { - if ( request.status === 200 && request.responseText ) { - try { - var json = JSON.parse( request.responseText ); - window.location = json.url; - } catch ( e ) { - console.error( e ); - } - } - }; - } ); - } ); -} )(); diff --git a/tests/runner/package.json b/tests/runner/package.json deleted file mode 100644 index bedb411a9..000000000 --- a/tests/runner/package.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "type": "module" -} diff --git a/tests/runner/queue.js b/tests/runner/queue.js deleted file mode 100644 index 1c9ac1acb..000000000 --- a/tests/runner/queue.js +++ /dev/null @@ -1,119 +0,0 @@ -import chalk from "chalk"; -import { getBrowserString } from "./lib/getBrowserString.js"; -import { - checkLastTouches, - createBrowserWorker, - restartBrowser, - setBrowserWorkerUrl -} from "./browsers.js"; - -const TEST_POLL_TIMEOUT = 1000; - -const queue = []; - -export function getNextBrowserTest( reportId ) { - const index = queue.findIndex( ( test ) => test.id === reportId ); - if ( index === -1 ) { - return; - } - - // Remove the completed test from the queue - const previousTest = queue[ index ]; - queue.splice( index, 1 ); - - // Find the next test for the same browser - for ( const test of queue.slice( index ) ) { - if ( test.fullBrowser === previousTest.fullBrowser ) { - - // Set the URL for our tracking - setBrowserWorkerUrl( test.browser, test.url ); - test.running = true; - - // Return the URL for the next test. - // listeners.js will use this to set the browser URL. - return { url: test.url }; - } - } -} - -export function retryTest( reportId, maxRetries ) { - if ( !maxRetries ) { - return; - } - const test = queue.find( ( test ) => test.id === reportId ); - if ( test ) { - test.retries++; - if ( test.retries <= maxRetries ) { - console.log( - `\nRetrying test ${ reportId } for ${ chalk.yellow( - test.options.suite - ) }...${ test.retries }` - ); - return test; - } - } -} - -export async function hardRetryTest( reportId, maxHardRetries ) { - if ( !maxHardRetries ) { - return false; - } - const test = queue.find( ( test ) => test.id === reportId ); - if ( test ) { - test.hardRetries++; - if ( test.hardRetries <= maxHardRetries ) { - console.log( - `\nHard retrying test ${ reportId } for ${ chalk.yellow( - test.options.suite - ) }...${ test.hardRetries }` - ); - await restartBrowser( test.browser ); - return true; - } - } - return false; -} - -export function addRun( url, browser, options ) { - queue.push( { - browser, - fullBrowser: getBrowserString( browser ), - hardRetries: 0, - id: options.reportId, - url, - options, - retries: 0, - running: false - } ); -} - -export async function runAll() { - return new Promise( async( resolve, reject ) => { - while ( queue.length ) { - try { - await checkLastTouches(); - } catch ( error ) { - reject( error ); - } - - // Run one test URL per browser at a time - const browsersTaken = []; - for ( const test of queue ) { - if ( browsersTaken.indexOf( test.fullBrowser ) > -1 ) { - continue; - } - browsersTaken.push( test.fullBrowser ); - if ( !test.running ) { - test.running = true; - try { - await createBrowserWorker( test.url, test.browser, test.options ); - } catch ( error ) { - reject( error ); - } - } - } - await new Promise( ( resolve ) => setTimeout( resolve, TEST_POLL_TIMEOUT ) ); - } - resolve(); - } ); -} diff --git a/tests/runner/reporter.js b/tests/runner/reporter.js deleted file mode 100644 index 6e47a68e4..000000000 --- a/tests/runner/reporter.js +++ /dev/null @@ -1,134 +0,0 @@ -import chalk from "chalk"; -import * as Diff from "diff"; -import { getBrowserString } from "./lib/getBrowserString.js"; -import { prettyMs } from "./lib/prettyMs.js"; - -function serializeForDiff( value ) { - - // Use naive serialization for everything except types with confusable values - if ( typeof value === "string" ) { - return JSON.stringify( value ); - } - if ( typeof value === "bigint" ) { - return `${ value }n`; - } - return `${ value }`; -} - -export function reportTest( test, reportId, { browser, headless } ) { - if ( test.status === "passed" ) { - - // Write to console without newlines - process.stdout.write( "." ); - return; - } - - let message = `${ chalk.bold( `${ test.suiteName }: ${ test.name }` ) }`; - message += `\nTest ${ test.status } on ${ chalk.yellow( - getBrowserString( browser, headless ) - ) } (${ chalk.bold( reportId ) }).`; - - // test.assertions only contains passed assertions; - // test.errors contains all failed asssertions - if ( test.errors.length ) { - for ( const error of test.errors ) { - message += "\n"; - if ( error.message ) { - message += `\n${ error.message }`; - } - message += `\n${ chalk.gray( error.stack ) }`; - - // Show expected and actual values - // if either is defined and non-null. - // error.actual is set to null for failed - // assert.expect() assertions, so skip those as well. - // This should be fine because error.expected would - // have to also be null for this to be skipped. - if ( error.expected != null || error.actual != null ) { - message += `\nexpected: ${ chalk.red( JSON.stringify( error.expected ) ) }`; - message += `\nactual: ${ chalk.green( JSON.stringify( error.actual ) ) }`; - let diff; - - if ( Array.isArray( error.expected ) && Array.isArray( error.actual ) ) { - - // Diff arrays - diff = Diff.diffArrays( error.expected, error.actual ); - } else if ( - typeof error.expected === "object" && - typeof error.actual === "object" - ) { - - // Diff objects - diff = Diff.diffJson( error.expected, error.actual ); - } else if ( - typeof error.expected === "number" && - typeof error.actual === "number" - ) { - - // Diff numbers directly - const value = error.actual - error.expected; - if ( value > 0 ) { - diff = [ { added: true, value: `+${ value }` } ]; - } else { - diff = [ { removed: true, value: `${ value }` } ]; - } - } else if ( - typeof error.expected === "string" && - typeof error.actual === "string" - ) { - - // Diff the characters of strings - diff = Diff.diffChars( error.expected, error.actual ); - } else { - - // Diff everything else as words - diff = Diff.diffWords( - serializeForDiff( error.expected ), - serializeForDiff( error.actual ) - ); - } - - if ( diff ) { - message += "\n"; - message += diff - .map( ( part ) => { - if ( part.added ) { - return chalk.green( part.value ); - } - if ( part.removed ) { - return chalk.red( part.value ); - } - return chalk.gray( part.value ); - } ) - .join( "" ); - } - } - } - } - - console.log( `\n\n${ message }` ); - - // Only return failed messages - if ( test.status === "failed" ) { - return message; - } -} - -export function reportEnd( result, reportId, { browser, headless, jquery, migrate, suite } ) { - const fullBrowser = getBrowserString( browser, headless ); - console.log( - `\n\nTests finished in ${ prettyMs( result.runtime ) } ` + - `for ${ chalk.yellow( suite ) } ` + - `and jQuery ${ chalk.yellow( jquery ) } ` + - ( migrate ? `with ${ chalk.yellow( "jQuery Migrate enabled " ) }` : "" ) + - `in ${ chalk.yellow( fullBrowser ) } (${ chalk.bold( reportId ) })...` - ); - console.log( - ( result.status !== "passed" ? - `${ chalk.red( result.testCounts.failed ) } failed. ` : - "" ) + - `${ chalk.green( result.testCounts.total ) } passed. ` + - `${ chalk.gray( result.testCounts.skipped ) } skipped.` - ); - return result.testCounts; -} diff --git a/tests/runner/run.js b/tests/runner/run.js deleted file mode 100644 index 9c4f8d479..000000000 --- a/tests/runner/run.js +++ /dev/null @@ -1,338 +0,0 @@ -import chalk from "chalk"; -import { asyncExitHook, gracefulExit } from "exit-hook"; -import { getLatestBrowser } from "./browserstack/api.js"; -import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js"; -import { localTunnel } from "./browserstack/local.js"; -import { reportEnd, reportTest } from "./reporter.js"; -import { createTestServer } from "./createTestServer.js"; -import { buildTestUrl } from "./lib/buildTestUrl.js"; -import { generateHash } from "./lib/generateHash.js"; -import { getBrowserString } from "./lib/getBrowserString.js"; -import { suites as allSuites } from "./flags/suites.js"; -import { cleanupAllBrowsers, touchBrowser } from "./browsers.js"; -import { - addRun, - getNextBrowserTest, - hardRetryTest, - retryTest, - runAll -} from "./queue.js"; - -const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000; - -/** - * Run test suites in parallel in different browser instances. - */ -export async function run( { - browser: browserNames = [], - browserstack, - concurrency, - debug, - hardRetries, - headless, - jquery: jquerys = [], - migrate, - retries = 0, - runId, - suite: suites = [], - verbose -} ) { - if ( !browserNames.length ) { - browserNames = [ "chrome" ]; - } - if ( !suites.length ) { - suites = allSuites; - } - if ( !jquerys.length ) { - jquerys = [ "3.7.1" ]; - } - if ( headless && debug ) { - throw new Error( - "Cannot run in headless mode and debug mode at the same time." - ); - } - - if ( verbose ) { - console.log( browserstack ? "Running in BrowserStack." : "Running locally." ); - } - - const errorMessages = []; - const pendingErrors = {}; - - // Convert browser names to browser objects - let browsers = browserNames.map( ( b ) => ( { browser: b } ) ); - const tunnelId = generateHash( - `${ Date.now() }-${ suites.join( ":" ) }-${ ( browserstack || [] ) - .concat( browserNames ) - .join( ":" ) }` - ); - - // A unique identifier for this run - if ( !runId ) { - runId = tunnelId; - } - - // Create the test app and - // hook it up to the reporter - const reports = Object.create( null ); - const app = await createTestServer( async( message ) => { - switch ( message.type ) { - case "testEnd": { - const reportId = message.id; - const report = reports[ reportId ]; - touchBrowser( report.browser ); - const errors = reportTest( message.data, reportId, report ); - pendingErrors[ reportId ] ??= Object.create( null ); - if ( errors ) { - pendingErrors[ reportId ][ message.data.name ] = errors; - } else { - const existing = pendingErrors[ reportId ][ message.data.name ]; - - // Show a message for flakey tests - if ( existing ) { - console.log(); - console.warn( - chalk.italic( - chalk.gray( existing.replace( "Test failed", "Test flakey" ) ) - ) - ); - console.log(); - delete pendingErrors[ reportId ][ message.data.name ]; - } - } - break; - } - case "runEnd": { - const reportId = message.id; - const report = reports[ reportId ]; - touchBrowser( report.browser ); - const { failed, total } = reportEnd( - message.data, - message.id, - reports[ reportId ] - ); - report.total = total; - - // Handle failure - if ( failed ) { - const retry = retryTest( reportId, retries ); - - // Retry if retryTest returns a test - if ( retry ) { - return retry; - } - - // Return early if hardRetryTest returns true - if ( await hardRetryTest( reportId, hardRetries ) ) { - return; - } - errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) ); - } - - // Run the next test - return getNextBrowserTest( reportId ); - } - case "ack": { - const report = reports[ message.id ]; - touchBrowser( report.browser ); - break; - } - default: - console.warn( "Received unknown message type:", message.type ); - } - } ); - - // Start up local test server - let server; - let port; - await new Promise( ( resolve ) => { - - // Pass 0 to choose a random, unused port - server = app.listen( 0, () => { - port = server.address().port; - resolve(); - } ); - } ); - - if ( !server || !port ) { - throw new Error( "Server not started." ); - } - - if ( verbose ) { - console.log( `Server started on port ${ port }.` ); - } - - function stopServer() { - return new Promise( ( resolve ) => { - server.close( () => { - if ( verbose ) { - console.log( "Server stopped." ); - } - resolve(); - } ); - } ); - } - - async function cleanup() { - console.log( "Cleaning up..." ); - - await cleanupAllBrowsers( { verbose } ); - - if ( tunnel ) { - await tunnel.stop(); - if ( verbose ) { - console.log( "Stopped BrowserStackLocal." ); - } - } - } - - asyncExitHook( - async() => { - await cleanup(); - await stopServer(); - }, - { wait: EXIT_HOOK_WAIT_TIMEOUT } - ); - - // Start up BrowserStackLocal - let tunnel; - if ( browserstack ) { - if ( headless ) { - console.warn( - chalk.italic( - "BrowserStack does not support headless mode. Running in normal mode." - ) - ); - headless = false; - } - - // Convert browserstack to browser objects. - // If browserstack is an empty array, fall back - // to the browsers array. - if ( browserstack.length ) { - browsers = browserstack.map( ( b ) => { - if ( !b ) { - return browsers[ 0 ]; - } - return buildBrowserFromString( b ); - } ); - } - - // Fill out browser defaults - browsers = await Promise.all( - browsers.map( async( browser ) => { - - // Avoid undici connect timeout errors - await new Promise( ( resolve ) => setTimeout( resolve, 100 ) ); - - const latestMatch = await getLatestBrowser( browser ); - if ( !latestMatch ) { - console.error( - chalk.red( `Browser not found: ${ getBrowserString( browser ) }.` ) - ); - gracefulExit( 1 ); - } - return latestMatch; - } ) - ); - - tunnel = await localTunnel( tunnelId ); - if ( verbose ) { - console.log( "Started BrowserStackLocal." ); - } - } - - function queueRuns( suite, browser ) { - const fullBrowser = getBrowserString( browser, headless ); - - for ( const jquery of jquerys ) { - const reportId = generateHash( `${ suite } ${ jquery } ${ fullBrowser }` ); - reports[ reportId ] = { browser, headless, jquery, migrate, suite }; - - const url = buildTestUrl( suite, { - browserstack, - jquery, - migrate, - port, - reportId - } ); - - const options = { - browserstack, - concurrency, - debug, - headless, - jquery, - migrate, - reportId, - runId, - suite, - tunnelId, - verbose - }; - - addRun( url, browser, options ); - } - } - - for ( const browser of browsers ) { - for ( const suite of suites ) { - queueRuns( [ suite ], browser ); - } - } - - try { - console.log( `Starting Run ${ runId }...` ); - await runAll(); - } catch ( error ) { - console.error( error ); - if ( !debug ) { - gracefulExit( 1 ); - } - } finally { - console.log(); - if ( errorMessages.length === 0 ) { - let stop = false; - for ( const report of Object.values( reports ) ) { - if ( !report.total ) { - stop = true; - console.error( - chalk.red( - `No tests were run for ${ report.suite } in ${ getBrowserString( - report.browser - ) }` - ) - ); - } - } - if ( stop ) { - return gracefulExit( 1 ); - } - console.log( chalk.green( "All tests passed!" ) ); - - if ( !debug || browserstack ) { - gracefulExit( 0 ); - } - } else { - console.error( chalk.red( `${ errorMessages.length } tests failed.` ) ); - console.log( - errorMessages.map( ( error, i ) => `\n${ i + 1 }. ${ error }` ).join( "\n" ) - ); - - if ( debug ) { - console.log(); - if ( browserstack ) { - console.log( "Leaving browsers with failures open for debugging." ); - console.log( - "View running sessions at https://automate.browserstack.com/dashboard/v2/" - ); - } else { - console.log( "Leaving browsers open for debugging." ); - } - console.log( "Press Ctrl+C to exit." ); - } else { - gracefulExit( 1 ); - } - } - } -} diff --git a/tests/runner/selenium/createDriver.js b/tests/runner/selenium/createDriver.js deleted file mode 100644 index 095c12214..000000000 --- a/tests/runner/selenium/createDriver.js +++ /dev/null @@ -1,84 +0,0 @@ -import { Builder, Capabilities, logging } from "selenium-webdriver"; -import Chrome from "selenium-webdriver/chrome.js"; -import Edge from "selenium-webdriver/edge.js"; -import Firefox from "selenium-webdriver/firefox.js"; -import { browserSupportsHeadless } from "../lib/getBrowserString.js"; - -// Set script timeout to 10min -const DRIVER_SCRIPT_TIMEOUT = 1000 * 60 * 10; - -export default async function createDriver( { browserName, headless, url, verbose } ) { - const capabilities = Capabilities[ browserName ](); - const prefs = new logging.Preferences(); - prefs.setLevel( logging.Type.BROWSER, logging.Level.ALL ); - capabilities.setLoggingPrefs( prefs ); - - let driver = new Builder().withCapabilities( capabilities ); - - const chromeOptions = new Chrome.Options(); - chromeOptions.addArguments( "--enable-chrome-browser-cloud-management" ); - - // Alter the chrome binary path if - // the CHROME_BIN environment variable is set - if ( process.env.CHROME_BIN ) { - if ( verbose ) { - console.log( `Setting chrome binary to ${ process.env.CHROME_BIN }` ); - } - chromeOptions.setChromeBinaryPath( process.env.CHROME_BIN ); - } - - const firefoxOptions = new Firefox.Options(); - - if ( process.env.FIREFOX_BIN ) { - if ( verbose ) { - console.log( `Setting firefox binary to ${ process.env.FIREFOX_BIN }` ); - } - - firefoxOptions.setBinary( process.env.FIREFOX_BIN ); - } - - const edgeOptions = new Edge.Options(); - edgeOptions.addArguments( "--enable-chrome-browser-cloud-management" ); - - // Alter the edge binary path if - // the EDGE_BIN environment variable is set - if ( process.env.EDGE_BIN ) { - if ( verbose ) { - console.log( `Setting edge binary to ${ process.env.EDGE_BIN }` ); - } - edgeOptions.setEdgeChromiumBinaryPath( process.env.EDGE_BIN ); - } - - if ( headless ) { - chromeOptions.addArguments( "--headless=new" ); - firefoxOptions.addArguments( "--headless" ); - edgeOptions.addArguments( "--headless=new" ); - if ( !browserSupportsHeadless( browserName ) ) { - console.log( - `Headless mode is not supported for ${ browserName }.` + - "Running in normal mode instead." - ); - } - } - - driver = await driver - .setChromeOptions( chromeOptions ) - .setFirefoxOptions( firefoxOptions ) - .setEdgeOptions( edgeOptions ) - .build(); - - if ( verbose ) { - const driverCapabilities = await driver.getCapabilities(); - const name = driverCapabilities.getBrowserName(); - const version = driverCapabilities.getBrowserVersion(); - console.log( `\nDriver created for ${ name } ${ version }` ); - } - - // Increase script timeout to 10min - await driver.manage().setTimeouts( { script: DRIVER_SCRIPT_TIMEOUT } ); - - // Set the first URL for the browser - await driver.get( url ); - - return driver; -} diff --git a/tests/runner/server.js b/tests/runner/server.js deleted file mode 100644 index 10fbc220f..000000000 --- a/tests/runner/server.js +++ /dev/null @@ -1,13 +0,0 @@ -import { createTestServer } from "./createTestServer.js"; - -const port = process.env.PORT || 3000; - -async function runServer() { - const app = await createTestServer(); - - app.listen( { port, host: "0.0.0.0" }, function() { - console.log( `Open tests at http://localhost:${ port }/tests/` ); - } ); -} - -runServer(); -- cgit v1.2.3