Improve webpack configs (deterministic builds across devices)

issue/311
Kelvin Schoofs 3 years ago
parent 1af2739d7b
commit 3d1aff327b

@ -15,6 +15,7 @@
"**/.gitattributes": true, "**/.gitattributes": true,
"**/.eslintcache": true, "**/.eslintcache": true,
"**/webpack.config.js": true, "**/webpack.config.js": true,
"webpack.plugin.js": true,
"**/tslint.json": true, "**/tslint.json": true,
"**/tsconfig.json": true "**/tsconfig.json": true
}, },

52
.vscode/tasks.json vendored

@ -37,52 +37,12 @@
"cwd": "./webview" "cwd": "./webview"
}, },
"group": "build", "group": "build",
"problemMatcher": [ "problemMatcher": {
{ "base": "$ts-webpack-watch",
"source": "parser", "source": "webpack-ts-loader",
"owner": "react", "owner": "webpack-ts-loader",
"fileLocation": "absolute", "applyTo": "allDocuments"
"applyTo": "allDocuments", },
"pattern": [
{
"regexp": "^SyntaxError: (.*): (.+) \\((\\d+):(\\d+)\\)$",
"file": 1,
"message": 2,
"line": 3,
"column": 4
}
],
"background": {
"activeOnStart": true,
"beginsPattern": "^Compiling.*",
"endsPattern": "^(Compiled successfully|Failed to compile)"
}
},
{
"source": "typescript",
"owner": "react",
"fileLocation": "absolute",
"applyTo": "allDocuments",
"pattern": [
{
"regexp": "^TypeScript error in (.*)\\((\\d+),(\\d+)\\):",
"file": 1,
"line": 2,
"column": 3
},
{
"regexp": "^(.{5,})$",
"message": 1,
"loop": true
}
],
"background": {
"activeOnStart": true,
"beginsPattern": "^Compiling.*",
"endsPattern": "^(Compiled successfully|Failed to compile)"
}
}
],
"isBackground": true "isBackground": true
} }
] ]

@ -1,12 +1,30 @@
# Changelog # Changelog
## Unreleased
### Development changes
- Webpack setup has been improved quite a bit, mostly to clean up long ugly paths and make builds deterministic:
- The custom `ProblemMatcherReporter` plugin is moved to `/webpack.plugin.js` and renamed to `WebpackPlugin`
- Now both webpack configs (extension and webview) make use of this plugin
- The plugin has the ability to remap module names/paths, accounting for several things:
- Paths in the global `/Yarn/Berry/` folder are now displayed as `/yarn/` and are simplified for easier reading
- Paths in the local `.yarn` folder get the same treatment as global ones, but using `.yarn/` as the prefix
- Other paths that are located within the (config's) project are made relative to the (config's) project root
- The plugin enhances the stats printer to use the clean simplified paths instead of e.g. `../../../Yarn/etc`
- The plugin handles generating chunk ids (`optimization.chunkIds` option)
- Acts mostly like a simplified version of the built-in `deterministic` option
- Uses the path remapping, resulting in paths not being different depending on where your global Yarn folder is
- These deterministic builds result in e.g. the same output chunk filenames
- Building the same commit on GitHub Actions or your own PC should result in e.g. the same source maps
- The `excludeModules` is now configured (and better handled) by the plugin
- The problem matcher for the `Extension Webview - Watch` task has been simplified and fixed due to the above change
## 1.22.0 (2021-09-21) ## 1.22.0 (2021-09-21)
### Fixes ### Fixes
- Partially fix issue with debug mode on code-server (05e1b69, #279) - Partially fix issue with debug mode on code-server (05e1b69, #279)
### Development changes ### Development changes
- I've added a `CHANGELOG.md` file to the repository containing the changelog for earlier versions. It'll contain already committed changes that have yet to be released. - I've added a `CHANGELOG.md` file to the repository containing the changelog for earlier versions. It'll contain already committed changes that have yet to be released.
- The extension now only enters debug mode when the environment variable `VSCODE_SSHFS_DEBUG` is the (case insensitive) string `"true"`. The `ExtensionContext.extensionMode` provided by Code does not influence this anymore. This is part due to #279, implemented in 05e1b69 which supersedes 48ef229. - The extension now only enters debug mode when the environment variable `VSCODE_SSHFS_DEBUG` is the (case insensitive) string `"true"`. The `ExtensionContext.extensionMode` provided by Code does not influence this anymore. This is part due to #279, implemented in 05e1b69 which supersedes 48ef229.

@ -5,6 +5,7 @@
const { join, resolve, dirname } = require('path'); const { join, resolve, dirname } = require('path');
const fs = require('fs'); const fs = require('fs');
const webpack = require('webpack'); const webpack = require('webpack');
const { WebpackPlugin } = require('./webpack.plugin');
/** /**
* @template T * @template T
@ -45,20 +46,6 @@ class CopyPuttyExecutable {
} }
} }
class ProblemMatcherReporter {
/**
* @param {webpack.Compiler} compiler
*/
apply(compiler) {
compiler.hooks.beforeCompile.tap('ProblemMatcherReporter-BeforeCompile', () => {
console.log('Compilation starting');
});
compiler.hooks.afterCompile.tap('ProblemMatcherReporter-AfterCompile', () => {
console.log('Compilation finished');
});
}
}
/**@type {webpack.Configuration}*/ /**@type {webpack.Configuration}*/
const config = { const config = {
mode: 'development', mode: 'development',
@ -95,7 +82,7 @@ const config = {
}, },
plugins: [ plugins: [
new CopyPuttyExecutable(), new CopyPuttyExecutable(),
new ProblemMatcherReporter(), new WebpackPlugin(),
], ],
optimization: { optimization: {
splitChunks: { splitChunks: {
@ -114,18 +101,6 @@ const config = {
modules: true, modules: true,
groupModulesByPath: true, groupModulesByPath: true,
modulesSpace: 50, modulesSpace: 50,
excludeModules(name, { issuerPath }) {
if (name.startsWith('external ')) return true;
const issuer = issuerPath && (issuerPath[issuerPath.length - 1].name || '').replace(/\\/g, '/');
if (!issuer) return false;
if (issuer.startsWith('./.yarn/')) return true;
if (issuer.startsWith('../')) {
const lower = issuer.toLowerCase();
if (lower.includes('/yarn/berry/cache/')) return true;
if (lower.includes('/.yarn/berry/cache/')) return true;
}
return false;
},
}, },
} }

@ -0,0 +1,131 @@
//@ts-check
'use strict';
const webpack = require('webpack');
const { createHash } = require('crypto');
class WebpackPlugin {
_formatIdCache = new Map();
/** @type {(id: string, rootPath: string) => string} */
formatId(id, rootPath) {
// Make sure all paths use /
id = id.replace(/\\/g, '/');
// For `[path]` we unwrap, format then rewrap
if (id[0] === '[' && id.endsWith(']')) {
return `[${this.formatId(id.slice(1, id.length - 1), rootPath)}]`;
}
// When dealing with `path1!path2`, format each segment separately
if (id.includes('!')) {
id = id.split('!').map(s => this.formatId(s, rootPath)).join('!');
}
// Make the paths relative to the project's rooth path if possible
if (id.startsWith(rootPath)) {
id = id.slice(rootPath.length);
id = (id[0] === '/' ? '.' : './') + id;
}
let formatted = this._formatIdCache.get(id);
if (formatted) return formatted;
// Check if we're dealing with a Yarn directory
let match = id.match(/^.*\/(\.?Yarn\/Berry|\.yarn)\/(.*)$/i);
if (!match) {
this._formatIdCache.set(id, formatted = id);
return formatted;
}
const [, yarn, filepath] = match;
// Check if we can extract the package name/version from the path
match = filepath.match(/^unplugged\/([^/]+?)\-[\da-f]{10}\/node_modules\/(.*)$/i)
|| filepath.match(/^cache\/([^/]+?)\-[\da-f]{10}\-\d+\.zip\/node_modules\/(.*)$/i);
if (!match) {
formatted = `/${yarn.toLowerCase() === '.yarn' ? '.' : ''}yarn/${filepath}`;
this._formatIdCache.set(id, formatted);
return formatted;
}
const [, name, path] = match;
formatted = `${yarn.toLowerCase() === '.yarn' ? '.' : '/'}yarn/${name}/${path}`;
this._formatIdCache.set(id, formatted);
return formatted;
}
_hashModuleCache = new Map();
/** @type {(mod: webpack.Module, rootPath: string) => string} */
hashModule(mod, rootPath) {
// Prefer `nameForCondition()` as it usually gives the actual file path
// while `identifier()` can have extra `!` or `|` suffixes, i.e. a hash that somehow differs between devices
const identifier = this.formatId(mod.nameForCondition() || mod.identifier(), rootPath);
let hash = this._hashModuleCache.get(identifier);
if (hash) return hash;
hash = createHash('sha1').update(identifier).digest('hex');
this._hashModuleCache.set(identifier, hash);
return hash;
}
/** @param {webpack.Compiler} compiler */
apply(compiler) {
// Output start/stop messages making the $ts-webpack-watch problemMatcher (provided by an extension) work
compiler.hooks.beforeCompile.tap('WebpackPlugin-BeforeCompile', () => {
console.log('Compilation starting');
});
compiler.hooks.afterCompile.tap('WebpackPlugin-AfterCompile', () => {
console.log('Compilation finished');
});
compiler.hooks.compilation.tap('WebpackPlugin-Compilation', compilation => {
const rootPath = (compilation.options.context || '').replace(/\\/g, '/');
compilation.options.optimization.chunkIds = false;
// Format `../../../Yarn/Berry/` with all the `cache`/`unplugged`/`__virtual__` to be more readable
// (i.e. `/yarn/package-npm-x.y.z/package/index.js` for global Yarn cache or `/.yarn/...` for local)
compilation.hooks.statsPrinter.tap('WebpackPlugin-StatsPrinter', stats => {
/** @type {(id: string | {}, context: any) => string} */
const tapModId = (id, context) => typeof id === 'string' ? this.formatId(context.formatModuleId(id), rootPath) : '???';
stats.hooks.print.for('module.name').tap('WebpackPlugin-ModuleName', tapModId);
});
// Include an `excludeModules` to `options.stats` to exclude modules loaded by dependencies
compilation.hooks.statsNormalize.tap('WebpackPlugin-StatsNormalize', stats => {
(stats.excludeModules || (stats.excludeModules = [])).push((name, { issuerPath }) => {
if (name.startsWith('external "')) return true;
const issuer = issuerPath && (issuerPath[issuerPath.length - 1].name || '').replace(/\\/g, '/');
if (!issuer) return false;
const lower = this.formatId(issuer, rootPath).toLowerCase();
if (lower.startsWith('/yarn/')) return true;
if (lower.startsWith('.yarn/')) return true;
return false;
});
});
// Determines how chunk IDs are generated, which is now actually deterministic
// (we make sure to clean Yarn paths to prevent issues with `../../Yarn/Berry` being different on devices)
compilation.hooks.chunkIds.tap('WebpackPlugin-ChunkIds', chunks => {
const chunkIds = new Map();
const overlapMap = new Set();
let minLength = 4; // show at least 3 characters
// Calculate the hashes for all the chunks
for (const chunk of chunks) {
if (chunk.id) {
console.log(`Chunk ${chunk.id} already has an ID`);
}
// We're kinda doing something similar to Webpack 5's DeterministicChunkIdsPlugin but different
const modules = compilation.chunkGraph.getChunkRootModules(chunk);
const hashes = modules.map(m => this.hashModule(m, rootPath)).sort();
const hasher = createHash('sha1');
for (const hash of hashes) hasher.update(hash);
const hash = hasher.digest('hex');
// With a 160-bit value, a clash is very unlikely, but let's check anyway
if (chunkIds.has(hash)) throw new Error('Hash collision for chunk IDs');
chunkIds.set(chunk, hash);
chunk.id = hash;
// Make sure the minLength remains high enough to avoid collisions
for (let i = minLength; i < hash.length; i++) {
const part = hash.slice(0, i);
if (overlapMap.has(part)) continue;
overlapMap.add(part);
minLength = i;
break;
}
}
// Assign the shortened (collision-free) hashes for all the chunks
for (const [chunk, hash] of chunkIds) {
chunk.id = hash.slice(0, minLength);
chunk.ids = [chunk.id];
}
});
});
}
}
module.exports.WebpackPlugin = WebpackPlugin;

@ -10,6 +10,7 @@ const MiniCssExtractPlugin = require('mini-css-extract-plugin');
const CssMinimizerPlugin = require('css-minimizer-webpack-plugin'); const CssMinimizerPlugin = require('css-minimizer-webpack-plugin');
const ESLintWebpackPlugin = require('eslint-webpack-plugin'); const ESLintWebpackPlugin = require('eslint-webpack-plugin');
const ReactRefreshWebpackPlugin = require('@pmmmwh/react-refresh-webpack-plugin'); const ReactRefreshWebpackPlugin = require('@pmmmwh/react-refresh-webpack-plugin');
const { WebpackPlugin } = require('../webpack.plugin');
require('dotenv').config(); require('dotenv').config();
@ -133,6 +134,7 @@ module.exports = (env, options) => {
options.serve && new webpack.HotModuleReplacementPlugin(), options.serve && new webpack.HotModuleReplacementPlugin(),
options.serve && new ReactRefreshWebpackPlugin(), options.serve && new ReactRefreshWebpackPlugin(),
new webpack.DefinePlugin(options.env), new webpack.DefinePlugin(options.env),
new WebpackPlugin(),
isEnvProduction && new MiniCssExtractPlugin({ isEnvProduction && new MiniCssExtractPlugin({
filename: 'static/css/[name].[contenthash:8].css', filename: 'static/css/[name].[contenthash:8].css',
chunkFilename: 'static/css/[name].[contenthash:8].chunk.css', chunkFilename: 'static/css/[name].[contenthash:8].chunk.css',
@ -167,6 +169,15 @@ module.exports = (env, options) => {
'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Origin': '*',
}, },
}, },
stats: {
ids: true,
assets: false,
chunks: false,
entrypoints: true,
modules: true,
groupModulesByPath: true,
modulesSpace: 50,
},
}; };
return config; return config;
}; };

Loading…
Cancel
Save