chore: Add webpack plugin to properly extract licenses used in compiled assets

This will create proper extracted license information for assets and stores it in `fist/file.js.license`.

Signed-off-by: Ferdinand Thiessen <opensource@fthiessen.de>

3 files changed, 232 insertions, 8 deletions

diff --git a/build/WebpackSPDXPlugin.js b/build/WebpackSPDXPlugin.js
new file mode 100644
index 00000000000..e9eeccca0f1
--- /dev/null
+++ b/build/WebpackSPDXPlugin.js
@@ -0,0 +1,216 @@
+"use strict";
+
+/**
+ * Party inspired by https://github.com/FormidableLabs/webpack-stats-plugin
+ *
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: MIT
+ */
+
+const { constants } = require('node:fs')
+const fs = require('node:fs/promises')
+const path = require('node:path')
+const webpack = require('webpack')
+
+class WebpackSPDXPlugin {
+	#options
+
+	/**
+	 * @param {object} opts Parameters
+	 * @param {Record<string, string>} opts.override Override licenses for packages
+	 */
+	constructor(opts = {}) {
+		this.#options = { override: {}, ...opts }
+	}
+
+	apply(compiler) {
+		compiler.hooks.thisCompilation.tap("spdx-plugin", (compilation) => {
+			// `processAssets` is one of the last hooks before frozen assets.
+			// We choose `PROCESS_ASSETS_STAGE_REPORT` which is the last possible
+			// stage after which to emit.
+			compilation.hooks.processAssets.tapPromise(
+				{
+					name: "spdx-plugin",
+					stage: compilation.constructor.PROCESS_ASSETS_STAGE_REPORT
+				},
+				() => this.emitLicenses(compilation)
+			)
+		})
+	}
+
+	/**
+	 * Find the nearest package.json
+	 * @param {string} dir Directory to start checking
+	 */
+	async #findPackage(dir) {
+		if (!dir || dir === '/' || dir === '.') {
+			return null
+		}
+
+		const packageJson = `${dir}/package.json`
+		try {
+			await fs.access(packageJson, constants.F_OK)
+		} catch (e) {
+			return await this.#findPackage(path.dirname(dir))
+		}
+
+		const { private: isPrivatePacket, name } = JSON.parse(await fs.readFile(packageJson))
+		// "private" is set in internal package.json which should not be resolved but the parent package.json
+		// Same if no name is set in package.json
+		if (isPrivatePacket === true || !name) {
+			return (await this.#findPackage(path.dirname(dir))) ?? packageJson
+		}
+		return packageJson
+	}
+
+	/**
+	 * 
+	 * @param {webpack.Compilation} compilation 
+	 * @param {*} callback 
+	 * @returns 
+	 */
+	async emitLicenses(compilation, callback) {
+		const moduleNames = (module) => module.modules?.map(moduleNames) ?? [module.name]
+		const logger = compilation.getLogger('spdx-plugin')
+		// cache the node packages
+		const packageInformation = new Map()
+
+		const warnings = new Set()
+		/** @type {Map<string, Set<webpack.Chunk>>} */
+		const sourceMap = new Map()
+
+		for (const chunk of compilation.chunks) {
+			for (const file of chunk.files) {
+				if (sourceMap.has(file)) {
+					sourceMap.get(file).add(chunk)
+				} else {
+					sourceMap.set(file, new Set([chunk]))
+				}
+			}
+		}
+
+		for (const [asset, chunks] of sourceMap.entries()) {
+			/** @type {Set<webpack.Module>} */
+			const modules = new Set()
+			/**
+			 * @param {webpack.Module} module 
+			 */
+			const addModule = (module) => {
+				if (module && !modules.has(module)) {
+					modules.add(module)
+					for (const dep of module.dependencies) {
+						addModule(compilation.moduleGraph.getModule(dep))
+					}
+				}
+			}
+			chunks.forEach((chunk) => chunk.getModules().forEach(addModule))
+
+			const sources = [...modules].map((module) => module.identifier())
+				.map((source) => {
+					const skipped = [
+						'delegated',
+						'external',
+						'container entry',
+						'ignored',
+						'remote',
+						'data:',
+					]
+					// Webpack sources that we can not infer license information or that is not included (external modules)
+					if (skipped.some((prefix) => source.startsWith(prefix))) {
+						return ''
+					}
+					// Internal webpack sources
+					if (source.startsWith('webpack/runtime')) {
+						return require.resolve('webpack')
+					}
+					// Handle webpack loaders
+					if (source.includes('!')) {
+						return source.split('!').at(-1)
+					}
+					if (source.includes('|')) {
+						return source
+							.split('|')
+							.filter((s) => s.startsWith(path.sep))
+							.at(0)
+					}
+					return source
+				})
+				.filter((s) => !!s)
+				.map((s) => s.split('?', 2)[0])
+
+			// Skip assets without modules, these are emitted by webpack plugins
+			if (sources.length === 0) {
+				logger.warn(`Skipping ${asset} because it does not contain any source information`)
+				continue
+			}
+
+			/** packages used by the current asset
+			 * @type {Set<string>}
+			 */
+			const packages = new Set()
+
+			// packages is the list of packages used by the asset
+			for (const sourcePath of sources) {
+				const pkg = await this.#findPackage(path.dirname(sourcePath))
+				if (!pkg) {
+					logger.warn(`No package for source found (${sourcePath})`)
+					continue
+				}
+
+				if (!packageInformation.has(pkg)) {
+					// Get the information from the package
+					const { author: packageAuthor, name, version, license: packageLicense, licenses } = JSON.parse(await fs.readFile(pkg))
+					// Handle legacy packages
+					let license = !packageLicense  && licenses
+						? licenses.map((entry) => entry.type ?? entry).join(' OR ')
+						: packageLicense
+					if (license?.includes(' ') && !license?.startsWith('(')) {
+						license = `(${license})`
+					}
+					// Handle both object style and string style author
+					const author = typeof packageAuthor === 'object'
+						? `${packageAuthor.name}` + (packageAuthor.mail ? ` <${packageAuthor.mail}>` : '')
+						: packageAuthor ?? `${name} developers`
+
+					packageInformation.set(pkg, {
+						version,
+						// Fallback to directory name if name is not set
+						name: name ?? path.basename(path.dirname(pkg)),
+						author,
+						license,
+					})
+				}
+				packages.add(pkg)
+			}
+
+			let output = 'This file is generated from multiple sources. Included packages:\n'
+			const authors = new Set()
+			const licenses = new Set()
+			for (const packageName of [...packages].sort()) {
+				const pkg = packageInformation.get(packageName)
+				const license = this.#options.override[pkg.name] ?? pkg.license
+				// Emit warning if not already done
+				if (!license && !warnings.has(pkg.name)) {
+					logger.warn(`Missing license information for package ${pkg.name}, you should add it to the 'override' option.`)
+					warnings.add(pkg.name)
+				}
+				licenses.add(license || 'unknown')
+				authors.add(pkg.author)
+				output += `\n- ${pkg.name}\n\t- version: ${pkg.version}\n\t- license: ${license}`
+			}
+			output += `\n\nSPDX-License-Identifier: ${[...licenses].sort().join(' AND ')}\n`
+			output += [...authors].sort().map((author) => `SPDX-FileCopyrightText: ${author}`).join('\n');
+
+			compilation.emitAsset(
+				asset.split('?', 2)[0] + '.license',
+				new webpack.sources.RawSource(output),
+			)
+		}
+
+		if (callback) {
+			return void callback()
+		}
+	}
+}
+
+module.exports = WebpackSPDXPlugin;
diff --git a/package.json b/package.json
index 6464f832f89..11bbc63f9e1 100644
--- a/package.json
+++ b/package.json
@@ -2,6 +2,7 @@
   "name": "nextcloud",
   "version": "1.0.0",
   "description": "Nextcloud Server",
+  "author": "Nextcloud GmbH and Nextcloud contributors",
   "private": true,
   "directories": {
     "lib": "lib",
@@ -188,6 +189,7 @@
     "webpack": "^5.91.0",
     "webpack-cli": "^5.0.2",
     "webpack-merge": "^5.8.0",
+    "webpack-stats-plugin": "^1.1.3",
     "workbox-webpack-plugin": "^7.0.0"
   },
   "browserslist": [
diff --git a/webpack.common.js b/webpack.common.js
index aa937719f24..3b8da3d948a 100644
--- a/webpack.common.js
+++ b/webpack.common.js
@@ -4,14 +4,16 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 const { VueLoaderPlugin } = require('vue-loader')
+const { readFileSync } = require('fs')
 const path = require('path')
+
 const BabelLoaderExcludeNodeModulesExcept = require('babel-loader-exclude-node-modules-except')
 const webpack = require('webpack')
 const NodePolyfillPlugin = require('node-polyfill-webpack-plugin')
 const WorkboxPlugin = require('workbox-webpack-plugin')
+const WebpackSPDXPlugin = require('./build/WebpackSPDXPlugin.js')
 
 const modules = require('./webpack.modules.js')
-const { readFileSync } = require('fs')
 
 const appVersion = readFileSync('./version.php').toString().match(/OC_VersionString[^']+'([^']+)/)?.[1] ?? 'unknown'
 
@@ -152,14 +154,11 @@ module.exports = {
 				// Lazy load the Terser plugin
 				const TerserPlugin = require('terser-webpack-plugin')
 				new TerserPlugin({
-					extractComments: {
-						condition: /^\**!|@license|@copyright|SPDX-License-Identifier|SPDX-FileCopyrightText/i,
-						filename: (fileData) => {
-						  // The "fileData" argument contains object with "filename", "basename", "query" and "hash"
-						  return `${fileData.filename}.license${fileData.query}`
-						},
-					},
+					extractComments: false,
 					terserOptions: {
+						format: {
+							comments: false,
+						},
 						compress: {
 							passes: 2,
 						},
@@ -239,6 +238,13 @@ module.exports = {
 			resourceRegExp: /^\.\/locale$/,
 			contextRegExp: /moment\/min$/,
 		}),
+
+		// Generate reuse license files
+		new WebpackSPDXPlugin({
+			override: {
+				select2: 'MIT',
+			}
+		}),
 	],
 	externals: {
 		OC: 'OC',