From 284e02f7636d633d0bdd8f70785429d7de002617 Mon Sep 17 00:00:00 2001 From: Kelvin Schoofs Date: Fri, 24 Sep 2021 22:19:31 +0200 Subject: [PATCH] Improve webpack configs (deterministic builds across devices) --- .vscode/settings.json | 1 + .vscode/tasks.json | 52 ++------------- CHANGELOG.md | 18 +++++- webpack.config.js | 29 +-------- webpack.plugin.js | 131 ++++++++++++++++++++++++++++++++++++++ webview/webpack.config.js | 11 ++++ 6 files changed, 168 insertions(+), 74 deletions(-) create mode 100644 webpack.plugin.js diff --git a/.vscode/settings.json b/.vscode/settings.json index 6af6a84..916fa0d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -15,6 +15,7 @@ "**/.gitattributes": true, "**/.eslintcache": true, "**/webpack.config.js": true, + "webpack.plugin.js": true, "**/tslint.json": true, "**/tsconfig.json": true }, diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 248ae2e..a85aeb2 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -37,52 +37,12 @@ "cwd": "./webview" }, "group": "build", - "problemMatcher": [ - { - "source": "parser", - "owner": "react", - "fileLocation": "absolute", - "applyTo": "allDocuments", - "pattern": [ - { - "regexp": "^SyntaxError: (.*): (.+) \\((\\d+):(\\d+)\\)$", - "file": 1, - "message": 2, - "line": 3, - "column": 4 - } - ], - "background": { - "activeOnStart": true, - "beginsPattern": "^Compiling.*", - "endsPattern": "^(Compiled successfully|Failed to compile)" - } - }, - { - "source": "typescript", - "owner": "react", - "fileLocation": "absolute", - "applyTo": "allDocuments", - "pattern": [ - { - "regexp": "^TypeScript error in (.*)\\((\\d+),(\\d+)\\):", - "file": 1, - "line": 2, - "column": 3 - }, - { - "regexp": "^(.{5,})$", - "message": 1, - "loop": true - } - ], - "background": { - "activeOnStart": true, - "beginsPattern": "^Compiling.*", - "endsPattern": "^(Compiled successfully|Failed to compile)" - } - } - ], + "problemMatcher": { + "base": "$ts-webpack-watch", + "source": "webpack-ts-loader", + "owner": "webpack-ts-loader", + "applyTo": "allDocuments" + }, "isBackground": true } ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 411c006..0244ca2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,28 @@ # Changelog +## Development changes +- Webpack setup has been improved quite a bit, mostly to clean up long ugly paths and make builds deterministic: + - The custom `ProblemMatcherReporter` plugin is moved to `/webpack.plugin.js` and renamed to `WebpackPlugin` + - Now both webpack configs (extension and webview) make use of this plugin + - The plugin has the ability to remap module names/paths, accounting for several things: + - Paths in the global `/Yarn/Berry/` folder are now displayed as `/yarn/` and are simplified for easier reading + - Paths in the local `.yarn` folder get the same treatment as global ones, but using `.yarn/` as the prefix + - Other paths that are located within the (config's) project are made relative to the (config's) project root + - The plugin enhances the stats printer to use the clean simplified paths instead of e.g. `../../../Yarn/etc` + - The plugin handles generating chunk ids (`optimization.chunkIds` option) + - Acts mostly like a simplified version of the built-in `deterministic` option + - Uses the path remapping, resulting in paths not being different depending on where your global Yarn folder is + - These deterministic builds result in e.g. the same output chunk filenames + - Building the same commit on GitHub Actions or your own PC should result in e.g. the same source maps + - The `excludeModules` is now configured (and better handled) by the plugin +- The problem matcher for the `Extension Webview - Watch` task has been simplified and fixed due to the above change + ## 1.22.0 (2021-09-21) ### Fixes - Partially fix issue with debug mode on code-server (05e1b69, #279) - ### Development changes - I've added a `CHANGELOG.md` file to the repository containing the changelog for earlier versions. It'll contain already committed changes that have yet to be released. - The extension now only enters debug mode when the environment variable `VSCODE_SSHFS_DEBUG` is the (case insensitive) string `"true"`. The `ExtensionContext.extensionMode` provided by Code does not influence this anymore. This is part due to #279, implemented in 05e1b69 which supersedes 48ef229. diff --git a/webpack.config.js b/webpack.config.js index 55c0da5..bc885d0 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -5,6 +5,7 @@ const { join, resolve, dirname } = require('path'); const fs = require('fs'); const webpack = require('webpack'); +const { WebpackPlugin } = require('./webpack.plugin'); /** * @template T @@ -45,20 +46,6 @@ class CopyPuttyExecutable { } } -class ProblemMatcherReporter { - /** - * @param {webpack.Compiler} compiler - */ - apply(compiler) { - compiler.hooks.beforeCompile.tap('ProblemMatcherReporter-BeforeCompile', () => { - console.log('Compilation starting'); - }); - compiler.hooks.afterCompile.tap('ProblemMatcherReporter-AfterCompile', () => { - console.log('Compilation finished'); - }); - } -} - /**@type {webpack.Configuration}*/ const config = { mode: 'development', @@ -95,7 +82,7 @@ const config = { }, plugins: [ new CopyPuttyExecutable(), - new ProblemMatcherReporter(), + new WebpackPlugin(), ], optimization: { splitChunks: { @@ -114,18 +101,6 @@ const config = { modules: true, groupModulesByPath: true, modulesSpace: 50, - excludeModules(name, { issuerPath }) { - if (name.startsWith('external ')) return true; - const issuer = issuerPath && (issuerPath[issuerPath.length - 1].name || '').replace(/\\/g, '/'); - if (!issuer) return false; - if (issuer.startsWith('./.yarn/')) return true; - if (issuer.startsWith('../')) { - const lower = issuer.toLowerCase(); - if (lower.includes('/yarn/berry/cache/')) return true; - if (lower.includes('/.yarn/berry/cache/')) return true; - } - return false; - }, }, } diff --git a/webpack.plugin.js b/webpack.plugin.js new file mode 100644 index 0000000..d746ce1 --- /dev/null +++ b/webpack.plugin.js @@ -0,0 +1,131 @@ + +//@ts-check +'use strict'; + +const webpack = require('webpack'); +const { createHash } = require('crypto'); + +class WebpackPlugin { + _formatIdCache = new Map(); + /** @type {(id: string, rootPath: string) => string} */ + formatId(id, rootPath) { + // Make sure all paths use / + id = id.replace(/\\/g, '/'); + // For `[path]` we unwrap, format then rewrap + if (id[0] === '[' && id.endsWith(']')) { + return `[${this.formatId(id.slice(1, id.length - 1), rootPath)}]`; + } + // When dealing with `path1!path2`, format each segment separately + if (id.includes('!')) { + id = id.split('!').map(s => this.formatId(s, rootPath)).join('!'); + } + // Make the paths relative to the project's rooth path if possible + if (id.startsWith(rootPath)) { + id = id.slice(rootPath.length); + id = (id[0] === '/' ? '.' : './') + id; + } + let formatted = this._formatIdCache.get(id); + if (formatted) return formatted; + // Check if we're dealing with a Yarn directory + let match = id.match(/^.*\/(\.?Yarn\/Berry|\.yarn)\/(.*)$/i); + if (!match) { + this._formatIdCache.set(id, formatted = id); + return formatted; + } + const [, yarn, filepath] = match; + // Check if we can extract the package name/version from the path + match = filepath.match(/^unplugged\/([^/]+?)\-[\da-f]{10}\/node_modules\/(.*)$/i) + || filepath.match(/^cache\/([^/]+?)\-[\da-f]{10}\-\d+\.zip\/node_modules\/(.*)$/i); + if (!match) { + formatted = `/${yarn.toLowerCase() === '.yarn' ? '.' : ''}yarn/${filepath}`; + this._formatIdCache.set(id, formatted); + return formatted; + } + const [, name, path] = match; + formatted = `${yarn.toLowerCase() === '.yarn' ? '.' : '/'}yarn/${name}/${path}`; + this._formatIdCache.set(id, formatted); + return formatted; + } + _hashModuleCache = new Map(); + /** @type {(mod: webpack.Module, rootPath: string) => string} */ + hashModule(mod, rootPath) { + // Prefer `nameForCondition()` as it usually gives the actual file path + // while `identifier()` can have extra `!` or `|` suffixes, i.e. a hash that somehow differs between devices + const identifier = this.formatId(mod.nameForCondition() || mod.identifier(), rootPath); + let hash = this._hashModuleCache.get(identifier); + if (hash) return hash; + hash = createHash('sha1').update(identifier).digest('hex'); + this._hashModuleCache.set(identifier, hash); + return hash; + } + /** @param {webpack.Compiler} compiler */ + apply(compiler) { + // Output start/stop messages making the $ts-webpack-watch problemMatcher (provided by an extension) work + compiler.hooks.beforeCompile.tap('WebpackPlugin-BeforeCompile', () => { + console.log('Compilation starting'); + }); + compiler.hooks.afterCompile.tap('WebpackPlugin-AfterCompile', () => { + console.log('Compilation finished'); + }); + compiler.hooks.compilation.tap('WebpackPlugin-Compilation', compilation => { + const rootPath = (compilation.options.context || '').replace(/\\/g, '/'); + compilation.options.optimization.chunkIds = false; + // Format `../../../Yarn/Berry/` with all the `cache`/`unplugged`/`__virtual__` to be more readable + // (i.e. `/yarn/package-npm-x.y.z/package/index.js` for global Yarn cache or `/.yarn/...` for local) + compilation.hooks.statsPrinter.tap('WebpackPlugin-StatsPrinter', stats => { + /** @type {(id: string | {}, context: any) => string} */ + const tapModId = (id, context) => typeof id === 'string' ? this.formatId(context.formatModuleId(id), rootPath) : '???'; + stats.hooks.print.for('module.name').tap('WebpackPlugin-ModuleName', tapModId); + }); + // Include an `excludeModules` to `options.stats` to exclude modules loaded by dependencies + compilation.hooks.statsNormalize.tap('WebpackPlugin-StatsNormalize', stats => { + (stats.excludeModules || (stats.excludeModules = [])).push((name, { issuerPath }) => { + if (name.startsWith('external "')) return true; + const issuer = issuerPath && (issuerPath[issuerPath.length - 1].name || '').replace(/\\/g, '/'); + if (!issuer) return false; + const lower = this.formatId(issuer, rootPath).toLowerCase(); + if (lower.startsWith('/yarn/')) return true; + if (lower.startsWith('.yarn/')) return true; + return false; + }); + }); + // Determines how chunk IDs are generated, which is now actually deterministic + // (we make sure to clean Yarn paths to prevent issues with `../../Yarn/Berry` being different on devices) + compilation.hooks.chunkIds.tap('WebpackPlugin-ChunkIds', chunks => { + const chunkIds = new Map(); + const overlapMap = new Set(); + let minLength = 4; // show at least 3 characters + // Calculate the hashes for all the chunks + for (const chunk of chunks) { + if (chunk.id) { + console.log(`Chunk ${chunk.id} already has an ID`); + } + // We're kinda doing something similar to Webpack 5's DeterministicChunkIdsPlugin but different + const modules = compilation.chunkGraph.getChunkRootModules(chunk); + const hashes = modules.map(m => this.hashModule(m, rootPath)).sort(); + const hasher = createHash('sha1'); + for (const hash of hashes) hasher.update(hash); + const hash = hasher.digest('hex'); + // With a 160-bit value, a clash is very unlikely, but let's check anyway + if (chunkIds.has(hash)) throw new Error('Hash collision for chunk IDs'); + chunkIds.set(chunk, hash); + chunk.id = hash; + // Make sure the minLength remains high enough to avoid collisions + for (let i = minLength; i < hash.length; i++) { + const part = hash.slice(0, i); + if (overlapMap.has(part)) continue; + overlapMap.add(part); + minLength = i; + break; + } + } + // Assign the shortened (collision-free) hashes for all the chunks + for (const [chunk, hash] of chunkIds) { + chunk.id = hash.slice(0, minLength); + chunk.ids = [chunk.id]; + } + }); + }); + } +} +module.exports.WebpackPlugin = WebpackPlugin; diff --git a/webview/webpack.config.js b/webview/webpack.config.js index 44e1763..706d5fb 100644 --- a/webview/webpack.config.js +++ b/webview/webpack.config.js @@ -10,6 +10,7 @@ const MiniCssExtractPlugin = require('mini-css-extract-plugin'); const CssMinimizerPlugin = require('css-minimizer-webpack-plugin'); const ESLintWebpackPlugin = require('eslint-webpack-plugin'); const ReactRefreshWebpackPlugin = require('@pmmmwh/react-refresh-webpack-plugin'); +const { WebpackPlugin } = require('../webpack.plugin'); require('dotenv').config(); @@ -133,6 +134,7 @@ module.exports = (env, options) => { options.serve && new webpack.HotModuleReplacementPlugin(), options.serve && new ReactRefreshWebpackPlugin(), new webpack.DefinePlugin(options.env), + new WebpackPlugin(), isEnvProduction && new MiniCssExtractPlugin({ filename: 'static/css/[name].[contenthash:8].css', chunkFilename: 'static/css/[name].[contenthash:8].chunk.css', @@ -167,6 +169,15 @@ module.exports = (env, options) => { 'Access-Control-Allow-Origin': '*', }, }, + stats: { + ids: true, + assets: false, + chunks: false, + entrypoints: true, + modules: true, + groupModulesByPath: true, + modulesSpace: 50, + }, }; return config; };