diff options
| -rw-r--r-- | test/runner/browserstack/browsers.js | 18 | ||||
| -rw-r--r-- | test/runner/browserstack/queue.js | 2 | ||||
| -rw-r--r-- | test/runner/run.js | 29 |
3 files changed, 29 insertions, 20 deletions
diff --git a/test/runner/browserstack/browsers.js b/test/runner/browserstack/browsers.js index 957c9aac8..3a7da4fc9 100644 --- a/test/runner/browserstack/browsers.js +++ b/test/runner/browserstack/browsers.js @@ -187,13 +187,19 @@ export async function cleanupAllBrowsers( { verbose } ) { const workersRemaining = Object.values( workers ); const numRemaining = workersRemaining.length; if ( numRemaining ) { - await Promise.all( - workersRemaining.map( ( worker ) => deleteWorker( worker.id ) ) - ); - if ( verbose ) { - console.log( - `Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.` + try { + await Promise.all( + workersRemaining.map( ( worker ) => deleteWorker( worker.id ) ) ); + if ( verbose ) { + console.log( + `Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.` + ); + } + } catch ( error ) { + + // Log the error, but do not consider the test run failed + console.error( error ); } } } diff --git a/test/runner/browserstack/queue.js b/test/runner/browserstack/queue.js index 10ef14a2b..c948f29bf 100644 --- a/test/runner/browserstack/queue.js +++ b/test/runner/browserstack/queue.js @@ -39,7 +39,7 @@ export function retryTest( reportId, maxRetries ) { console.log( `Retrying test ${ reportId } for ${ chalk.yellow( test.options.modules.join( ", " ) - ) }...` + ) }...${ test.retries }` ); return test; } diff --git a/test/runner/run.js b/test/runner/run.js index 2c90863b0..9f7a38aa7 100644 --- a/test/runner/run.js +++ b/test/runner/run.js @@ -83,7 +83,19 @@ export async function run( { if ( errors ) { pendingErrors[ reportId ][ message.data.name ] = errors; } else { - delete pendingErrors[ reportId ][ message.data.name ]; + const existing = pendingErrors[ reportId ][ message.data.name ]; + + // Show a message for flakey tests + if ( existing ) { + console.log(); + console.warn( + chalk.italic( + chalk.gray( existing.replace( "Test failed", "Test flakey" ) ) + ) + ); + console.log(); + delete pendingErrors[ reportId ][ message.data.name ]; + } } break; } @@ -103,24 +115,15 @@ export async function run( { // Handle failure if ( failed ) { const retry = retryTest( reportId, retries ); + + // Retry if retryTest returns a test if ( retry ) { return retry; } errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) ); - return getNextBrowserTest( reportId ); } - // Handle success - if ( - pendingErrors[ reportId ] && - Object.keys( pendingErrors[ reportId ] ).length - ) { - console.warn( "Detected flaky tests:" ); - for ( const [ , error ] in Object.entries( pendingErrors[ reportId ] ) ) { - console.warn( chalk.italic( chalk.gray( error ) ) ); - } - delete pendingErrors[ reportId ]; - } + // Run the next test return getNextBrowserTest( reportId ); } case "ack": { |