Build: migrate most grunt tasks off of grunt

Updated tasks include:

- lint
- npmcopy
- build, minify, and process for distribution.
- new custom build command using yargs
- compare size of minified/gzip built files
- pretest scripts, including qunit-fixture, babel transpilation, and npmcopy
- node smoke tests
- promises aplus tests
- new watch task using `rollup.watch` directly

Also:

- upgraded husky and added the new lint command
- updated lint config to use new "flat" config format. See https://eslint.org/docs/latest/use/configure/configuration-files-new
- Temporarily disabled one lint rule until flat config is supported by eslint-plugin-import. See https://github.com/import-js/eslint-plugin-import/issues/2556
- committed package-lock.json
- updated all test scripts to use the new build
- added an express test server that uses middleware-mockserver (this can be used to run tests without karma)
- build-all-variants is now build:all

Close gh-5318

1 files changed, 318 insertions, 312 deletions

diff --git a/build/tasks/build.js b/build/tasks/build.js
index 3fc37da8a..1a0d7d75a 100644
--- a/build/tasks/build.js
+++ b/build/tasks/build.js
@@ -6,350 +6,356 @@
 
 "use strict";
 
-module.exports = function( grunt ) {
-	const fs = require( "fs" );
-	const path = require( "path" );
-	const rollup = require( "rollup" );
-	const slimBuildFlags = require( "./lib/slim-build-flags" );
-	const rollupFileOverrides = require( "./lib/rollup-plugin-file-overrides" );
-	const srcFolder = path.resolve( `${ __dirname }/../../src` );
-	const read = function( fileName ) {
-		return grunt.file.read( `${ srcFolder }/${ fileName }` );
-	};
+const fs = require( "fs" );
+const path = require( "path" );
+const util = require( "util" );
+const exec = util.promisify( require( "child_process" ).exec );
+const rollup = require( "rollup" );
+const excludedFromSlim = require( "./lib/slim-exclude" );
+const rollupFileOverrides = require( "./lib/rollup-plugin-file-overrides" );
+const pkg = require( "../../package.json" );
+const isCleanWorkingDir = require( "./lib/isCleanWorkingDir" );
+const minify = require( "./minify" );
+const getTimestamp = require( "./lib/getTimestamp" );
+const verifyNodeVersion = require( "./lib/verifyNodeVersion" );
+const srcFolder = path.resolve( __dirname, "../../src" );
+
+const minimum = [ "core" ];
+
+// Exclude specified modules if the module matching the key is removed
+const removeWith = {
+	ajax: [ "manipulation/_evalUrl", "deprecated/ajax-event-alias" ],
+	callbacks: [ "deferred" ],
+	css: [ "effects", "dimensions", "offset" ],
+	"css/showHide": [ "effects" ],
+	deferred: {
+		remove: [ "ajax", "effects", "queue", "core/ready" ],
+		include: [ "core/ready-no-deferred" ]
+	},
+	event: [ "deprecated/ajax-event-alias", "deprecated/event" ],
+	selector: [ "css/hiddenVisibleSelectors", "effects/animatedSelector" ]
+};
 
-	const inputFileName = "jquery.js";
-	const inputRollupOptions = {
-		input: `${ srcFolder }/${ inputFileName }`
-	};
+async function read( filename ) {
+	return fs.promises.readFile( path.join( srcFolder, filename ), "utf8" );
+}
+
+// Remove the src folder and file extension
+// and ensure unix-style path separators
+function moduleName( filename ) {
+	return filename
+		.replace( `${srcFolder}${path.sep}`, "" )
+		.replace( /\.js$/, "" )
+		.split( path.sep )
+		.join( path.posix.sep );
+}
+
+async function readdirRecursive( dir, all = [] ) {
+	let files;
+	try {
+		files = await fs.promises.readdir( path.join( srcFolder, dir ), {
+			withFileTypes: true
+		} );
+	} catch ( e ) {
+		return all;
+	}
+	for ( const file of files ) {
+		const filepath = path.join( dir, file.name );
 
-	function getOutputRollupOptions( {
-		esm = false
-	} = {} ) {
-		const wrapperFileName = `wrapper${ esm ? "-esm" : "" }.js`;
+		if ( file.isDirectory() ) {
+			all.push( ...( await readdirRecursive( filepath ) ) );
+		} else {
+			all.push( moduleName( filepath ) );
+		}
+	}
+	return all;
+}
 
-		// Catch `// @CODE` and subsequent comment lines event if they don't start
-		// in the first column.
-		const wrapper = read( wrapperFileName )
-			.split( /[\x20\t]*\/\/ @CODE\n(?:[\x20\t]*\/\/[^\n]+\n)*/ );
+async function getOutputRollupOptions( { esm = false } = {} ) {
+	const wrapperFileName = `wrapper${esm ? "-esm" : ""}.js`;
+	const wrapperSource = await read( wrapperFileName );
 
-		return {
+	// Catch `// @CODE` and subsequent comment lines event if they don't start
+	// in the first column.
+	const wrapper = wrapperSource.split(
+		/[\x20\t]*\/\/ @CODE\n(?:[\x20\t]*\/\/[^\n]+\n)*/
+	);
 
-			// The ESM format is not actually used as we strip it during the
-			// build, inserting our own wrappers; it's just that it doesn't
-			// generate any extra wrappers so there's nothing for us to remove.
-			format: "esm",
+	return {
 
-			intro: `${ wrapper[ 0 ].replace( /\n*$/, "" ) }`,
-			outro: wrapper[ 1 ].replace( /^\n*/, "" )
-		};
-	}
+		// The ESM format is not actually used as we strip it during the
+		// build, inserting our own wrappers; it's just that it doesn't
+		// generate any extra wrappers so there's nothing for us to remove.
+		format: "esm",
 
-	const fileOverrides = new Map();
+		intro: wrapper[ 0 ].replace( /\n*$/, "" ),
+		outro: wrapper[ 1 ].replace( /^\n*/, "" )
+	};
+}
 
-	function getOverride( filePath ) {
-		return fileOverrides.get( path.resolve( filePath ) );
-	}
+const fileOverrides = new Map();
 
-	function setOverride( filePath, source ) {
+function setOverride( filePath, source ) {
 
-		// We want normalized paths in overrides as they will be matched
-		// against normalized paths in the file overrides Rollup plugin.
-		fileOverrides.set( path.resolve( filePath ), source );
-	}
+	// We want normalized paths in overrides as they will be matched
+	// against normalized paths in the file overrides Rollup plugin.
+	fileOverrides.set( path.resolve( filePath ), source );
+}
 
-	grunt.registerMultiTask(
-		"build",
-		"Build jQuery ECMAScript modules, " +
-			"(include/exclude modules with +/- flags), embed date/version",
-	async function() {
-		const done = this.async();
-
-		try {
-			const flags = this.flags;
-			const optIn = flags[ "*" ];
-			let name = grunt.option( "filename" );
-			const esm = !!grunt.option( "esm" );
-			const distFolder = grunt.option( "dist-folder" );
-			const minimum = this.data.minimum;
-			const removeWith = this.data.removeWith;
-			const excluded = [];
-			const included = [];
-			let version = grunt.config( "pkg.version" );
-
-			// We'll skip printing the whole big exclusions for a bare `build:*:*:slim` which
-			// usually comes from `custom:slim`.
-			const isPureSlim = !!( flags.slim && flags[ "*" ] &&
-				Object.keys( flags ).length === 2 );
-
-			delete flags[ "*" ];
-
-			if ( flags.slim ) {
-				delete flags.slim;
-				for ( const flag of slimBuildFlags ) {
-					flags[ flag ] = true;
-				}
-			}
+function unique( array ) {
+	return [ ...new Set( array ) ];
+}
 
+async function checkExclude( exclude, include ) {
+	const included = [ ...include ];
+	const excluded = [ ...exclude ];
 
-			/**
-			 * Recursively calls the excluder to remove on all modules in the list
-			 * @param {Array} list
-			 * @param {String} [prepend] Prepend this to the module name.
-			 *  Indicates we're walking a directory
-			 */
-			const excludeList = ( list, prepend ) => {
-				if ( list ) {
-					prepend = prepend ? `${ prepend }/` : "";
-					list.forEach( function( module ) {
-
-						// Exclude var modules as well
-						if ( module === "var" ) {
-							excludeList(
-								fs.readdirSync( `${ srcFolder }/${ prepend }${ module }` ),
-								prepend + module
-							);
-							return;
-						}
-						if ( prepend ) {
-
-							// Skip if this is not a js file and we're walking files in a dir
-							if ( !( module = /([\w-\/]+)\.js$/.exec( module ) ) ) {
-								return;
-							}
-
-							// Prepend folder name if passed
-							// Remove .js extension
-							module = prepend + module[ 1 ];
-						}
-
-						// Avoid infinite recursion
-						if ( excluded.indexOf( module ) === -1 ) {
-							excluder( "-" + module );
-						}
-					} );
-				}
-			};
-
-			/**
-			 * Adds the specified module to the excluded or included list, depending on the flag
-			 * @param {String} flag A module path relative to
-			 *  the src directory starting with + or - to indicate
-			 *  whether it should be included or excluded
-			 */
-			const excluder = flag => {
-				let additional;
-				const m = /^(\+|-|)([\w\/-]+)$/.exec( flag );
-				const exclude = m[ 1 ] === "-";
-				const module = m[ 2 ];
-
-				if ( exclude ) {
-
-					// Can't exclude certain modules
-					if ( minimum.indexOf( module ) === -1 ) {
-
-						// Add to excluded
-						if ( excluded.indexOf( module ) === -1 ) {
-							grunt.log.writeln( flag );
-							excluded.push( module );
-
-							// Exclude all files in the folder of the same name
-							// These are the removable dependencies
-							// It's fine if the directory is not there
-							try {
-
-								// `selector` is a special case as we don't just remove
-								// the module, but we replace it with `selector-native`
-								// which re-uses parts of the `src/selector` folder.
-								if ( module !== "selector" ) {
-									excludeList(
-										fs.readdirSync( `${ srcFolder }/${ module }` ),
-										module
-									);
-								}
-							} catch ( e ) {
-								grunt.verbose.writeln( e );
-							}
-						}
-
-						additional = removeWith[ module ];
-
-						// Check removeWith list
-						if ( additional ) {
-							excludeList( additional.remove || additional );
-							if ( additional.include ) {
-								included.push( ...additional.include );
-								grunt.log.writeln( "+" + additional.include );
-							}
-						}
-					} else {
-						grunt.log.error( "Module \"" + module + "\" is a minimum requirement." );
-					}
-				} else {
-					grunt.log.writeln( flag );
-					included.push( module );
-				}
-			};
-
-			// Filename can be passed to the command line using
-			// command line options
-			// e.g. grunt build --filename=jquery-custom.js
-			name = name ? `${ distFolder }/${ name }` : this.data.dest;
-
-			// append commit id to version
-			if ( process.env.COMMIT ) {
-				version += " " + process.env.COMMIT;
-			}
+	for ( const module of exclude ) {
+		if ( minimum.indexOf( module ) !== -1 ) {
+			throw new Error( `Module \"${module}\" is a minimum requirement.` );
+		}
 
-			// figure out which files to exclude based on these rules in this order:
-			//  dependency explicit exclude
-			//  > explicit exclude
-			//  > explicit include
-			//  > dependency implicit exclude
-			//  > implicit exclude
-			// examples:
-			//  *                  none (implicit exclude)
-			//  *:*                all (implicit include)
-			//  *:*:-css           all except css and dependents (explicit > implicit)
-			//  *:*:-css:+effects  same (excludes effects because explicit include is
-			//                     trumped by explicit exclude of dependency)
-			//  *:+effects         none except effects and its dependencies
-			//                     (explicit include trumps implicit exclude of dependency)
-			for ( const flag in flags ) {
-				excluder( flag );
-			}
+		// Exclude all files in the dir of the same name
+		// These are the removable dependencies
+		// It's fine if the directory is not there
+		// `selector` is a special case as we don't just remove
+		// the module, but we replace it with `selector-native`
+		// which re-uses parts of the `src/selector` dir.
+		if ( module !== "selector" ) {
+			const files = await readdirRecursive( module );
+			excluded.push( ...files );
+		}
 
-			// Remove the jQuery export from the entry file, we'll use our own
-			// custom wrapper.
-			setOverride( inputRollupOptions.input,
-				read( inputFileName ).replace( /\n*export \{ jQuery, jQuery as \$ };\n*/, "\n" ) );
-
-			// Replace exports/global with a noop noConflict
-			if ( excluded.includes( "exports/global" ) ) {
-				const index = excluded.indexOf( "exports/global" );
-				setOverride( `${ srcFolder }/exports/global.js`,
-					"import jQuery from \"../core.js\";\n\n" +
-						"jQuery.noConflict = function() {};" );
-				excluded.splice( index, 1 );
-			}
+		// Check removeWith list
+		const additional = removeWith[ module ];
+		if ( additional ) {
+			const [ additionalExcluded, additionalIncluded ] = await checkExclude(
+				additional.remove || additional,
+				additional.include || []
+			);
+			excluded.push( ...additionalExcluded );
+			included.push( ...additionalIncluded );
+		}
+	}
+
+	return [ unique( excluded ), unique( included ) ];
+}
+
+async function writeCompiled( { code, dir, filename, version } ) {
+	const compiledContents = code
+
+		// Embed Version
+		.replace( /@VERSION/g, version )
+
+		// Embed Date
+		// yyyy-mm-ddThh:mmZ
+		.replace( /@DATE/g, new Date().toISOString().replace( /:\d+\.\d+Z$/, "Z" ) );
+
+	await fs.promises.writeFile( path.join( dir, filename ), compiledContents );
+	console.log( `[${getTimestamp()}] ${filename} v${version} created.` );
+}
+
+// Build jQuery ECMAScript modules
+async function build( {
+	amd,
+	dir = "dist",
+	exclude = [],
+	filename = "jquery.js",
+	include = [],
+	esm = false,
+	slim = false,
+	version,
+	watch = false
+} = {} ) {
+	const pureSlim = slim && !exclude.length && !include.length;
+
+	// Add the short commit hash to the version string
+	// when the version is not for a release.
+	if ( !version ) {
+		const { stdout } = await exec( "git rev-parse --short HEAD" );
+		const isClean = await isCleanWorkingDir();
+
+		// "+[slim.]SHA" is semantically correct
+		// Add ".dirty" as well if the working dir is not clean
+		version = `${pkg.version}+${slim ? "slim." : ""}${stdout.trim()}${isClean ? "" : ".dirty"}`;
+	} else if ( slim ) {
+		version += "+slim";
+	}
+
+	await fs.promises.mkdir( dir, { recursive: true } );
+
+	// Exclude slim modules when slim is true
+	const [ excluded, included ] = await checkExclude(
+		slim ? exclude.concat( excludedFromSlim ) : exclude,
+		include
+	);
+
+	// Replace exports/global with a noop noConflict
+	if ( excluded.includes( "exports/global" ) ) {
+		const index = excluded.indexOf( "exports/global" );
+		setOverride(
+			`${srcFolder}/exports/global.js`,
+			"import { jQuery } from \"../core.js\";\n\n" +
+				"jQuery.noConflict = function() {};"
+		);
+		excluded.splice( index, 1 );
+	}
 
-			// Set a desired AMD name.
-			let amdName = grunt.option( "amd" );
-			if ( amdName != null ) {
-				if ( amdName ) {
-					grunt.log.writeln( "Naming jQuery with AMD name: " + amdName );
-				} else {
-					grunt.log.writeln( "AMD name now anonymous" );
-				}
+	// Set a desired AMD name.
+	if ( amd != null ) {
+		if ( amd ) {
+			console.log( "Naming jQuery with AMD name: " + amd );
+		} else {
+			console.log( "AMD name now anonymous" );
+		}
+
+		// Replace the AMD name in the AMD export
+		// No name means an anonymous define
+		const amdExportContents = await read( "exports/amd.js" );
+		setOverride(
+			`${srcFolder}/exports/amd.js`,
+			amdExportContents.replace(
 
 				// Remove the comma for anonymous defines
-				setOverride( `${ srcFolder }/exports/amd.js`,
-					read( "exports/amd.js" )
-						.replace( /(\s*)"jquery"(,\s*)/,
-							amdName ? "$1\"" + amdName + "\"$2" : "" ) );
-			}
+				/(\s*)"jquery"(,\s*)/,
+				amd ? `$1\"${amd}\"$2` : " "
+			)
+		);
+	}
 
-			grunt.verbose.writeflags( excluded, "Excluded" );
-			grunt.verbose.writeflags( included, "Included" );
+	// Append excluded modules to version.
+	// Skip adding exclusions for slim builds.
+	// Don't worry about semver syntax for these.
+	if ( !pureSlim && excluded.length ) {
+		version += " -" + excluded.join( ",-" );
+	}
 
-			// Indicate a Slim build without listing all the exclusions
-			// to save space.
-			if ( isPureSlim ) {
-				version += " slim";
+	// Append extra included modules to version.
+	if ( !pureSlim && included.length ) {
+		version += " +" + included.join( ",+" );
+	}
 
-			// Append excluded modules to version.
-			} else if ( excluded.length ) {
-				version += " -" + excluded.join( ",-" );
-			}
+	const inputOptions = {
+		input: `${srcFolder}/jquery.js`
+	};
 
-			if ( excluded.length ) {
-
-				// Set pkg.version to version with excludes or with the "slim" marker,
-				// so minified file picks it up but skip the commit hash the same way
-				// it's done for the full build.
-				const commitlessVersion = version.replace( " " + process.env.COMMIT, "" );
-				grunt.config.set( "pkg.version", commitlessVersion );
-				grunt.verbose.writeln( "Version changed to " + commitlessVersion );
-
-				// Replace excluded modules with empty sources.
-				for ( const module of excluded ) {
-					setOverride(
-						`${ srcFolder }/${ module }.js`,
-
-						// The `selector` module is not removed, but replaced
-						// with `selector-native`.
-						module === "selector" ? read( "selector-native.js" ) : ""
-					);
-				}
-			}
+	const includedImports = included
+		.map( ( module ) => `import "./${module}.js";` )
+		.join( "\n" );
 
-			// Turn off opt-in if necessary
-			if ( !optIn ) {
+	const jQueryFileContents = await read( "jquery.js" );
+	if ( include.length ) {
 
-				// Remove the default inclusions, they will be overwritten with the explicitly
-				// included ones.
-				setOverride( inputRollupOptions.input, "" );
+		// If include is specified, only add those modules.
+		setOverride( inputOptions.input, includedImports );
+	} else {
 
-			}
+		// Remove the jQuery export from the entry file, we'll use our own
+		// custom wrapper.
+		setOverride(
+			inputOptions.input,
+			jQueryFileContents.replace( /\n*export \{ jQuery, jQuery as \$ };\n*/, "\n" ) +
+				includedImports
+		);
+	}
 
-			// Import the explicitly included modules.
-			if ( included.length ) {
-				setOverride( inputRollupOptions.input,
-					getOverride( inputRollupOptions.input ) + included
-						.map( module => `import "./${module}.js";` )
-						.join( "\n" ) );
-			}
+	// Replace excluded modules with empty sources.
+	for ( const module of excluded ) {
+		setOverride(
+			`${srcFolder}/${module}.js`,
 
-			const bundle = await rollup.rollup( {
-				...inputRollupOptions,
-				plugins: [ rollupFileOverrides( fileOverrides ) ]
-			} );
+			// The `selector` module is not removed, but replaced
+			// with `selector-native`.
+			module === "selector" ? await read( "selector-native.js" ) : ""
+		);
+	}
 
-			const outputRollupOptions =
-				getOutputRollupOptions( { esm } );
+	const bundle = await rollup.rollup( {
+		...inputOptions,
+		plugins: [ rollupFileOverrides( fileOverrides ) ]
+	} );
 
-			const { output: [ { code } ] } = await bundle.generate( outputRollupOptions );
+	const outputOptions = await getOutputRollupOptions( { esm } );
 
-			const compiledContents = code
+	if ( watch ) {
+		const watcher = rollup.watch( {
+			...inputOptions,
+			output: [ outputOptions ],
+			plugins: [ rollupFileOverrides( fileOverrides ) ],
+			watch: {
+				include: `${srcFolder}/**`,
+				skipWrite: true
+			}
+		} );
+
+		watcher.on( "event", async( event ) => {
+			switch ( event.code ) {
+				case "ERROR":
+					console.error( event.error );
+					break;
+				case "BUNDLE_END":
+					const {
+						output: [ { code } ]
+					} = await event.result.generate( outputOptions );
+
+					await writeCompiled( {
+						code,
+						dir,
+						filename,
+						version
+					} );
 
-				// Embed Version
-				.replace( /@VERSION/g, version )
+					await minify( { dir, filename, esm } );
+					break;
+			}
+		} );
 
-				// Embed Date
-				// yyyy-mm-ddThh:mmZ
-				.replace(
-					/@DATE/g,
-					( new Date() ).toISOString()
-						.replace( /:\d+\.\d+Z$/, "Z" )
-				);
+		return watcher;
+	} else {
+		const {
+			output: [ { code } ]
+		} = await bundle.generate( outputOptions );
 
-			grunt.file.write( name, compiledContents );
-			grunt.log.ok( `File '${ name }' created.` );
-			done();
-		} catch ( err ) {
-			done( err );
-		}
-	} );
+		await writeCompiled( { code, dir, filename, version } );
+		await minify( { dir, filename, esm } );
+	}
+}
+
+async function buildDefaultFiles( { version, watch } = {} ) {
+	await Promise.all( [
+		build( { version, watch } ),
+		build( { filename: "jquery.slim.js", slim: true, version, watch } ),
+		build( {
+			dir: "dist-module",
+			filename: "jquery.module.js",
+			esm: true,
+			version,
+			watch
+		} ),
+		build( {
+			dir: "dist-module",
+			filename: "jquery.slim.module.js",
+			esm: true,
+			slim: true,
+			version,
+			watch
+		} )
+	] );
+
+	// Earlier Node.js versions do not support the ESM format.
+	if ( !verifyNodeVersion() ) {
+		return;
+	}
 
-	// Special "alias" task to make custom build creation less grawlix-y
-	// Translation example
-	//
-	//   grunt custom:+ajax,-dimensions,-effects,-offset
-	//
-	// Becomes:
-	//
-	//   grunt build:*:*:+ajax:-dimensions:-effects:-offset
-	//
-	// There's also a special "slim" alias that resolves to the jQuery Slim build
-	// configuration:
-	//
-	//   grunt custom:slim
-	grunt.registerTask( "custom", function() {
-		const args = this.args;
-		const modules = args.length ?
-			args[ 0 ].split( "," ).join( ":" ) :
-			"";
-
-		grunt.log.writeln( "Creating custom build...\n" );
-		grunt.task.run( [ "build:*:*" + ( modules ? ":" + modules : "" ), "minify", "dist" ] );
+	const { compareSize } = await import( "./compare_size.mjs" );
+	return compareSize( {
+		files: [
+			"dist/jquery.min.js",
+			"dist/jquery.slim.min.js",
+			"dist-module/jquery.module.min.js",
+			"dist-module/jquery.slim.module.min.js"
+		]
 	} );
-};
+}
+
+module.exports = { build, buildDefaultFiles };