Tests: fix flakey message logs; ignore delete worker failures

Close gh-5431

diff --git a/test/runner/browserstack/browsers.js b/test/runner/browserstack/browsers.js
index 957c9aac8d2d79e3855790517cf46965bf020ee4..3a7da4fc9f0da459adf15788f435ced7b5b58b7f 100644 (file)
--- a/test/runner/browserstack/browsers.js
+++ b/test/runner/browserstack/browsers.js
@@ -187,13 +187,19 @@ export async function cleanupAllBrowsers( { verbose } ) {
        const workersRemaining = Object.values( workers );
        const numRemaining = workersRemaining.length;
        if ( numRemaining ) {
-               await Promise.all(
-                       workersRemaining.map( ( worker ) => deleteWorker( worker.id ) )
-               );
-               if ( verbose ) {
-                       console.log(
-                               `Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
+               try {
+                       await Promise.all(
+                               workersRemaining.map( ( worker ) => deleteWorker( worker.id ) )
                        );
+                       if ( verbose ) {
+                               console.log(
+                                       `Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
+                               );
+                       }
+               } catch ( error ) {
+
+                       // Log the error, but do not consider the test run failed
+                       console.error( error );
                }
        }
 }

diff --git a/test/runner/browserstack/queue.js b/test/runner/browserstack/queue.js
index 10ef14a2b844b39c81d9b3e16e01d3ce393f3b69..c948f29bf95542c29a278ac4ba5c88ca8e5dc03c 100644 (file)
--- a/test/runner/browserstack/queue.js
+++ b/test/runner/browserstack/queue.js
@@ -39,7 +39,7 @@ export function retryTest( reportId, maxRetries ) {
                        console.log(
                                `Retrying test ${ reportId } for ${ chalk.yellow(
                                        test.options.modules.join( ", " )
-                               ) }...`
+                               ) }...${ test.retries }`
                        );
                        return test;
                }

diff --git a/test/runner/run.js b/test/runner/run.js
index adf1d33fb55923331488e07d68d24ca196981bf9..1dfa24cbad8fbf79edf3a0dc728b2104e6e5a528 100644 (file)
--- a/test/runner/run.js
+++ b/test/runner/run.js
@@ -83,7 +83,19 @@ export async function run( {
                                if ( errors ) {
                                        pendingErrors[ reportId ][ message.data.name ] = errors;
                                } else {
-                                       delete pendingErrors[ reportId ][ message.data.name ];
+                                       const existing = pendingErrors[ reportId ][ message.data.name ];
+
+                                       // Show a message for flakey tests
+                                       if ( existing ) {
+                                               console.log();
+                                               console.warn(
+                                                       chalk.italic(
+                                                               chalk.gray( existing.replace( "Test failed", "Test flakey" ) )
+                                                       )
+                                               );
+                                               console.log();
+                                               delete pendingErrors[ reportId ][ message.data.name ];
+                                       }
                                }
                                break;
                        }
@@ -103,24 +115,15 @@ export async function run( {
                                // Handle failure
                                if ( failed ) {
                                        const retry = retryTest( reportId, retries );
+
+                                       // Retry if retryTest returns a test
                                        if ( retry ) {
                                                return retry;
                                        }
                                        errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
-                                       return getNextBrowserTest( reportId );
                                }
 
-                               // Handle success
-                               if (
-                                       pendingErrors[ reportId ] &&
-                                       Object.keys( pendingErrors[ reportId ] ).length
-                               ) {
-                                       console.warn( "Detected flaky tests:" );
-                                       for ( const [ , error ] in Object.entries( pendingErrors[ reportId ] ) ) {
-                                               console.warn( chalk.italic( chalk.gray( error ) ) );
-                                       }
-                                       delete pendingErrors[ reportId ];
-                               }
+                               // Run the next test
                                return getNextBrowserTest( reportId );
                        }
                        case "ack": {