Refactor how configuration files are loaded

pull/373/head
Kelvin Schoofs 2 years ago
parent 11b8f052db
commit 831a247f4b

@ -23,6 +23,12 @@
- Write operations are: `createDirectory`, `writeFile`, `delete`, and `rename`
- Since `readDirectory`, `readFile` and `stat` are disabled by default, it should prevent extension detection spam (see #341)
- Added the `SHELL_CONFIG` flag to force a specific remote shell configuration (#331)
- Refactored how (and also from where) configuration files are loaded
- The extension is now better at splitting up where it loads configs from into layers (global, workspace, ...)
- When settings change, only the appropriate layer (e.g. a workspace folder) is reloaded, instead of reloading everything
- Loading config files from the VS Code settings in remote workspaces is now supported
- All layers, including (remote) workspace folders should fully support the `sshfs.configpaths` setting
- Although this can change, for workspace folders, paths specified in the global/workspace settings are also scanned
### Development changes
- Move the whole flag system from config.ts to flags.ts

@ -1,11 +1,13 @@
import { ConfigLocation, FileSystemConfig, invalidConfigName, parseConnectionString } from 'common/fileSystemConfig';
import { readFile, writeFile } from 'fs';
import { parse as parseJsonc, ParseError } from 'jsonc-parser';
import * as semver from 'semver';
import { ParseError, parse as parseJsonc, printParseErrorCode } from 'jsonc-parser';
import * as path from 'path';
import * as vscode from 'vscode';
import { MANAGER } from './extension';
import { Logging } from './logging';
import { catchingPromise, toPromise } from './utils';
import { catchingPromise } from './utils';
const fs = vscode.workspace.fs;
// Logger scope with default warning/error options (which enables stacktraces) disabled
const logging = Logging.scope(undefined, false);
@ -21,6 +23,7 @@ function randomAvailableName(configs: FileSystemConfig[], index = 0): [string, n
return [name, index + 1];
}
// TODO: Do this better, especially since we can dynamically start adding configs (for workspaceFolders)
export async function renameNameless() {
const conf = vscode.workspace.getConfiguration('sshfs');
const inspect = conf.inspect<FileSystemConfig[]>('configs')!;
@ -55,97 +58,136 @@ export function getConfigs() {
export const UPDATE_LISTENERS: ((configs: FileSystemConfig[]) => any)[] = [];
async function readConfigFile(location: string, shouldExist = false): Promise<FileSystemConfig[]> {
const content = await toPromise<Buffer>(cb => readFile(location, cb)).catch((e: NodeJS.ErrnoException) => e);
async function readConfigFile(file: vscode.Uri, quiet: boolean): Promise<FileSystemConfig[] | undefined> {
const content = await fs.readFile(file).then<Uint8Array | NodeJS.ErrnoException>(v => v, e => e);
if (content instanceof Error) {
if (content.code === 'ENOENT' && !shouldExist) return [];
logging.error`Error while reading config file ${location}: ${content}`;
return [];
if (content.code === 'ENOENT' && quiet) return undefined;
logging.error`Error while reading config file ${file}: ${content}`;
return undefined;
}
const errors: ParseError[] = [];
const parsed: FileSystemConfig[] | null = parseJsonc(content.toString(), errors);
const parsed: FileSystemConfig[] | null = parseJsonc(Buffer.from(content.buffer).toString(), errors);
if (!parsed || errors.length) {
logging.error`Couldn't parse ${location} as a 'JSON with Comments' file`;
vscode.window.showErrorMessage(`Couldn't parse ${location} as a 'JSON with Comments' file`);
const formatted = errors.map(({ error, offset, length }) => `${printParseErrorCode(error)} at ${offset}-${offset + length}`);
logging.error`Couldn't parse ${file} due to invalid JSON:\n${formatted.join('\n')}`;
vscode.window.showErrorMessage(`Couldn't parse the SSH FS config file at ${file}, invalid JSON`);
return [];
}
parsed.forEach(c => c._locations = [c._location = location]);
logging.debug`Read ${parsed.length} configs from ${location}`;
parsed.forEach(c => c._locations = [c._location = file.toString()]);
logging.debug`Read ${parsed.length} configs from ${file}`;
return parsed;
}
export function getConfigLocations(): ConfigLocation[] {
// Fetch configs from vscode settings
const config = vscode.workspace.getConfiguration('sshfs');
const configpaths = { workspace: [] as string[], global: [] as string[] };
if (config) {
const inspect2 = config.inspect<string[]>('configpaths')!;
configpaths.workspace = inspect2.workspaceValue || [];
configpaths.global = inspect2.globalValue || [];
async function readConfigDirectory(uri: vscode.Uri, quiet: boolean): Promise<FileSystemConfig[] | undefined> {
const stat = await fs.stat(uri).then(e => e, () => undefined);
if (!stat) return undefined;
const files = await fs.readDirectory(uri); // errors if not a directory
logging.debug`readConfigDirectory got files: ${files}`;
const parsed = await Promise.all(files
.filter(([, t]) => t & vscode.FileType.File).map(([f]) => f)
.filter(file => file.endsWith('.json') || file.endsWith('.jsonc'))
.map(file => readConfigFile(vscode.Uri.joinPath(uri, file), quiet)));
return parsed.some(Boolean) ? parsed.filter(Array.isArray).flat() : undefined;
}
const skipDisconnectedUri = (uri: vscode.Uri) => uri.scheme === 'ssh' && !MANAGER?.connectionManager.getActiveConnection(uri.authority);
async function findConfigs(uri: vscode.Uri, quiet: boolean): Promise<FileSystemConfig[] | undefined> {
if (uri.scheme === 'ssh') {
// Ignore SSH URIs for connections that are still connecting
if (skipDisconnectedUri(uri)) {
logging.debug`Skipping config file '${uri}' for disconnected config`;
return [];
}
}
try {
return await readConfigDirectory(uri, quiet);
} catch {
return await readConfigFile(uri, quiet);
}
return [...configpaths.workspace, ...configpaths.global];
}
export async function loadConfigsRaw(): Promise<FileSystemConfig[]> {
logging.info('Loading configurations...');
await renameNameless();
// Keep all found configs "ordened" by layer, for proper deduplication/merging
const layered = {
folder: [] as FileSystemConfig[],
workspace: [] as FileSystemConfig[],
global: [] as FileSystemConfig[],
/**
* Tries to read all configs from all possible locations matching the given location.
* This function will report errors to the user/logger, and never reject. An empty array may be returned.
* This function might read multiple files when given a path to a directory, and will aggregate the results.
* Will return `undefined` if the given file doesn't exist, or lead to a directory with no readable config files.
* Will return an empty array if the given path is a relative path.
*/
async function findConfigFiles(location: string | vscode.Uri, quiet = false): Promise<[configs: FileSystemConfig[] | undefined, isAbsolute: boolean]> {
if (location instanceof vscode.Uri) {
return [await findConfigs(location, quiet), true];
} else if (location.match(/^([a-zA-Z0-9+.-]+):/)) {
return [await findConfigs(vscode.Uri.parse(location), quiet), true];
} else if (path.isAbsolute(location)) {
return [await findConfigs(vscode.Uri.file(location), quiet), true];
}
return [[], false];
}
async function tryFindConfigFiles(location: string | vscode.Uri, source: string): Promise<FileSystemConfig[]> {
const [found, isAbsolute] = await findConfigFiles(location, true);
if (found) return found;
logging[isAbsolute ? 'error' : 'info']`No configs found in '${location}' provided by ${source}`;
return [];
}
function getConfigPaths(scope?: vscode.WorkspaceFolder): Record<'global' | 'workspace' | 'folder', string[]> {
const config = vscode.workspace.getConfiguration('sshfs', scope);
const inspect = config.inspect<string[]>('configpaths')!;
return {
global: inspect.globalValue || [],
workspace: inspect.workspaceValue || [],
folder: inspect.workspaceFolderValue || [],
};
// Fetch configs from vscode settings
}
let configLayers: {
global: FileSystemConfig[];
workspace: FileSystemConfig[];
folder: Map<string, FileSystemConfig[]>;
};
/** Only loads `sshfs.configs` into `configLayers`, ignoring `sshfs.configpaths` */
async function loadGlobalOrWorkspaceConfigs(): Promise<void> {
const config = vscode.workspace.getConfiguration('sshfs');
const configpaths = { workspace: [] as string[], global: [] as string[] };
if (config) {
const inspect = config.inspect<FileSystemConfig[]>('configs')!;
// Note: workspaceFolderValue not used here, we do it later for all workspace folders
layered.workspace = inspect.workspaceValue || [];
layered.global = inspect.globalValue || [];
layered.workspace.forEach(c => c._locations = [c._location = vscode.ConfigurationTarget.Workspace]);
layered.global.forEach(c => c._locations = [c._location = vscode.ConfigurationTarget.Global]);
// Get all sshfs.configpaths values into an array
const inspect2 = config.inspect<string[]>('configpaths')!;
configpaths.workspace = inspect2.workspaceValue || [];
configpaths.global = inspect2.globalValue || [];
}
// Fetch configs from config files
for (const location of configpaths.workspace) {
layered.workspace = [
...layered.workspace,
...await readConfigFile(location, true),
];
configLayers.global = inspect.globalValue || [];
configLayers.workspace = inspect.workspaceValue || [];
configLayers.global.forEach(c => c._locations = [c._location = vscode.ConfigurationTarget.Global]);
configLayers.workspace.forEach(c => c._locations = [c._location = vscode.ConfigurationTarget.Workspace]);
}
/** Loads `sshfs.configs` and (including global/workspace-provided) relative `sshfs.configpaths` into `configLayers` */
async function loadWorkspaceFolderConfigs(folder: vscode.WorkspaceFolder): Promise<FileSystemConfig[]> {
if (skipDisconnectedUri(folder.uri)) {
configLayers.folder.set(folder.uri.toString(), []);
return [];
}
for (const location of configpaths.global) {
layered.global = [
...layered.global,
...await readConfigFile(location, true),
];
const config = vscode.workspace.getConfiguration('sshfs', folder).inspect<FileSystemConfig[]>('configs');
const configs = config && config.workspaceFolderValue || [];
if (configs.length) {
logging.debug`Read ${configs.length} configs from workspace folder ${folder.uri}`;
configs.forEach(c => c._locations = [c._location = `WorkspaceFolder ${folder.uri}`]);
}
// Fetch configs from opened folders (workspaces)
// Should we really support workspace folders, and not just workspaces?
/*
const { workspaceFolders } = vscode.workspace;
if (workspaceFolders) {
for (const { uri } of workspaceFolders) {
if (uri.scheme !== 'file') continue;
const fConfig = vscode.workspace.getConfiguration('sshfs', uri).inspect<FileSystemConfig[]>('configs');
const fConfigs = fConfig && fConfig.workspaceFolderValue || [];
if (fConfigs.length) {
logging.debug`Read ${fConfigs.length} configs from workspace folder ${uri}`;
fConfigs.forEach(c => c._locations = [c._location = `WorkspaceFolder ${uri}`]);
}
layered.folder = [
...await readConfigFile(path.resolve(uri.fsPath, 'sshfs.json')),
...await readConfigFile(path.resolve(uri.fsPath, 'sshfs.jsonc')),
...fConfigs,
...layered.folder,
];
const configPaths = getConfigPaths(folder);
for (const location of [...configPaths.global, ...configPaths.workspace, ...configPaths.folder]) {
if (path.isAbsolute(location)) continue;
const uri = vscode.Uri.joinPath(folder.uri, location);
const found = await tryFindConfigFiles(uri, `WorkspaceFolder '${folder.uri}'`);
if (found) configs.push(...found);
}
}*/
// Store all configs in one array, in order of importance
const all = [...layered.folder, ...layered.workspace, ...layered.global];
configLayers.folder.set(folder.uri.toString(), configs);
return configs;
}
function applyConfigLayers(): void {
// Merge all layers into a single array of configs, in order of importance
const all: FileSystemConfig[] = [
...(vscode.workspace.workspaceFolders || []).flatMap(ws => configLayers.folder.get(ws.uri.toString()) || []),
...configLayers.workspace,
...configLayers.global,
];
all.forEach(c => c.name = (c.name || '').toLowerCase()); // It being undefined shouldn't happen, but better be safe
// Let the user do some cleaning with the raw configs
for (const conf of all) {
@ -171,49 +213,123 @@ export async function loadConfigsRaw(): Promise<FileSystemConfig[]> {
});
}
}
// After cleaning up, ignore the configurations that are still bad
return all.filter(c => !invalidConfigName(c.name));
}
export async function loadConfigs(): Promise<FileSystemConfig[]> {
const all = await loadConfigsRaw();
// Remove duplicates, merging those where the more specific config has `merge` set
// Folder comes before Workspace, comes before Global
const configs: FileSystemConfig[] = [];
for (const conf of all) {
const dup = configs.find(d => d.name === conf.name);
// Remove duplicates, merging those where the more specific config has `merge` set (in the order from above)
loadedConfigs = [];
for (const conf of all.filter(c => !invalidConfigName(c.name))) {
const dup = loadedConfigs.find(d => d.name === conf.name);
if (dup) {
if (dup.merge) {
// The folder settings should overwrite the higher up defined settings
// Since .sshfs.json gets read after vscode settings, these can overwrite configs
// of the same level, which I guess is a nice feature?
logging.debug`\tMerging duplicate ${conf.name} from ${conf._locations}`;
dup._locations = [...dup._locations, ...conf._locations];
Object.assign(dup, Object.assign(conf, dup));
Object.assign(dup, { ...conf, ...dup });
} else {
logging.debug`\tIgnoring duplicate ${conf.name} from ${conf._locations}`;
}
} else {
logging.debug`\tAdded configuration ${conf.name} from ${conf._locations}`;
configs.push(conf);
loadedConfigs.push(conf);
}
}
loadedConfigs = configs;
logging.info`Found ${loadedConfigs.length} configurations`;
logging.info`Applied config layers resulting in ${loadedConfigs.length} configurations`;
UPDATE_LISTENERS.forEach(listener => listener(loadedConfigs));
return loadedConfigs;
}
export let LOADING_CONFIGS: Promise<FileSystemConfig[]>;
export async function loadConfigs(): Promise<FileSystemConfig[]> {
return LOADING_CONFIGS = catchingPromise(async loaded => {
logging.info('Loading configurations...');
await renameNameless();
// Keep all found configs "ordened" by layer, for proper deduplication/merging
// while also allowing partially refreshing (workspaceFolder configs) without having to reload *everything*
configLayers = { global: [], workspace: [], folder: new Map() };
// Fetch global/workspace configs from vscode settings
loadGlobalOrWorkspaceConfigs();
// Fetch configs from config files defined in global/workspace settings
const configpaths = getConfigPaths();
for (const location of configpaths.global) {
configLayers.global.push(...await tryFindConfigFiles(location, 'Global Settings'));
}
for (const location of configpaths.workspace) {
configLayers.workspace.push(...await tryFindConfigFiles(location, 'Workspace Settings'));
}
// Fetch configs from opened folders
for (const folder of vscode.workspace.workspaceFolders || []) {
await loadWorkspaceFolderConfigs(folder);
}
applyConfigLayers();
loaded(loadedConfigs);
});
}
loadConfigs();
export async function reloadWorkspaceFolderConfigs(authority: string): Promise<void> {
authority = authority.toLowerCase();
const promises = (vscode.workspace.workspaceFolders || []).map(workspaceFolder => {
if (workspaceFolder.uri.authority.toLowerCase() !== authority) return;
logging.info`Reloading workspace folder configs for '${authority}' connection`;
return loadWorkspaceFolderConfigs(workspaceFolder);
});
if (!promises.length) return;
await Promise.all(promises);
applyConfigLayers();
}
vscode.workspace.onDidChangeConfiguration(async (e) => {
if (e.affectsConfiguration('sshfs.configpaths')) {
logging.info('Config paths changed for global/workspace, reloading configs...');
return loadConfigs();
}
let updatedGlobal = e.affectsConfiguration('sshfs.configs');
if (updatedGlobal) {
logging.info('Config paths changed for global/workspace, updating layers...');
await loadGlobalOrWorkspaceConfigs();
}
let updatedAtAll = updatedGlobal;
for (const workspaceFolder of vscode.workspace.workspaceFolders || []) {
if (updatedGlobal
|| e.affectsConfiguration('sshfs.configs', workspaceFolder)
|| e.affectsConfiguration('sshfs.configpaths', workspaceFolder)) {
logging.info(`Configs and/or config paths changed for workspace folder ${workspaceFolder.uri}, updating layers...`);
await loadWorkspaceFolderConfigs(workspaceFolder);
updatedAtAll = true;
}
}
if (updatedAtAll) applyConfigLayers();
});
vscode.workspace.onDidChangeWorkspaceFolders(event => {
LOADING_CONFIGS = catchingPromise<FileSystemConfig[]>(async loaded => {
logging.info('Workspace folders changed, recalculating configs with updated workspaceFolder configs...');
event.removed.forEach(folder => configLayers.folder.delete(folder.uri.toString()));
for (const folder of event.added) await loadWorkspaceFolderConfigs(folder);
applyConfigLayers();
loaded(loadedConfigs);
}).catch(e => {
logging.error`Error while reloading configs in onDidChangeWorkspaceFolders: ${e}`;
return loadedConfigs;
});
});
export type ConfigAlterer = (configs: FileSystemConfig[]) => FileSystemConfig[] | null | false;
export async function alterConfigs(location: ConfigLocation, alterer: ConfigAlterer) {
let uri!: vscode.Uri | undefined;
let prettyLocation: string | undefined;
if (typeof location === 'string' && location.startsWith('WorkspaceFolder ')) {
prettyLocation = location;
uri = vscode.Uri.parse(location.substring(16));
location = vscode.ConfigurationTarget.WorkspaceFolder;
}
switch (location) {
case vscode.ConfigurationTarget.WorkspaceFolder:
throw new Error(`Trying to update WorkspaceFolder settings with WorkspaceFolder Uri`);
case vscode.ConfigurationTarget.Global:
prettyLocation ||= 'Global';
case vscode.ConfigurationTarget.Workspace:
case vscode.ConfigurationTarget.WorkspaceFolder:
const conf = vscode.workspace.getConfiguration('sshfs');
prettyLocation ||= 'Workspace';
const conf = vscode.workspace.getConfiguration('sshfs', uri);
const inspect = conf.inspect<FileSystemConfig[]>('configs')!;
// If the array doesn't exist, create a new empty one
const array = [, inspect.globalValue, inspect.workspaceValue, inspect.workspaceFolderValue][location] || [];
const array = inspect[[, 'globalValue', 'workspaceValue', 'workspaceFolderValue'][location]!] || [];
let modified = alterer(array);
if (!modified) return;
modified = modified.map((config) => {
@ -224,11 +340,16 @@ export async function alterConfigs(location: ConfigLocation, alterer: ConfigAlte
return newConfig;
});
await conf.update('configs', modified, location);
logging.debug`\tUpdated configs in ${[, 'Global', 'Workspace', 'WorkspaceFolder'][location]} settings.json`;
logging.debug`\tUpdated configs in ${prettyLocation} Settings`;
return;
}
if (typeof location !== 'string') throw new Error(`Invalid _location field: ${location}`);
const configs = await readConfigFile(location, true);
uri = vscode.Uri.parse(location, true);
const configs = await readConfigFile(uri, true);
if (!configs) {
logging.error`Config file '${uri}' not found while altering configs'`;
throw new Error(`Config file '${uri}' not found while altering configs'`);
}
let altered = alterer(configs);
if (!altered) return;
altered = altered.map((config) => {
@ -238,12 +359,11 @@ export async function alterConfigs(location: ConfigLocation, alterer: ConfigAlte
}
return newConfig;
});
const data = JSON.stringify(altered, null, 4);
await toPromise(cb => writeFile(location, data, cb))
.catch((e: NodeJS.ErrnoException) => {
const data = Buffer.from(JSON.stringify(altered, null, 4));
try { await fs.writeFile(uri, data); } catch (e) {
logging.error`Error while writing configs to ${location}: ${e}`;
throw e;
});
}
logging.debug`\tWritten modified configs to ${location}`;
await loadConfigs();
}
@ -337,8 +457,3 @@ export function configMatches(a: FileSystemConfig, b: FileSystemConfig): boolean
// could just use === between the two configs. This'll do for now.
return valueMatches(a, b);
}
vscode.workspace.onDidChangeConfiguration(async (e) => {
// if (!e.affectsConfiguration('sshfs.configs')) return;
return loadConfigs();
});

@ -41,8 +41,9 @@ export class ConnectionManager {
/** Fired when a pending connection gets added/removed */
public readonly onPendingChanged = this.onPendingChangedEmitter.event;
public getActiveConnection(name: string, config?: FileSystemConfig): Connection | undefined {
const con = config && this.connections.find(con => configMatches(con.config, config));
return con || (config ? undefined : this.connections.find(con => con.config.name === name));
if (config) return this.connections.find(con => configMatches(con.config, config));
name = name.toLowerCase();
return this.connections.find(con => con.config.name === name);
}
public getActiveConnections(): Connection[] {
return [...this.connections];

@ -1,7 +1,7 @@
import type { FileSystemConfig } from 'common/fileSystemConfig';
import * as vscode from 'vscode';
import { loadConfigs } from './config';
import { loadConfigs, reloadWorkspaceFolderConfigs } from './config';
import type { Connection } from './connection';
import { FileSystemRouter } from './fileSystemRouter';
import { Logging, setDebug } from './logging';
@ -26,7 +26,10 @@ interface CommandHandler {
handleTerminal?(terminal: SSHPseudoTerminal): void;
}
export async function activate(context: vscode.ExtensionContext) {
/** `findConfigs` in config.ts ignores URIs for still-connecting connections */
export let MANAGER: Manager | undefined;
export function activate(context: vscode.ExtensionContext) {
Logging.info`Extension activated, version ${getVersion()}, mode ${context.extensionMode}`;
Logging.debug`Running VS Code version ${vscode.version} ${process.versions}`;
@ -47,12 +50,7 @@ export async function activate(context: vscode.ExtensionContext) {
// I really don't like having to pass context to *everything*, so let's do it this way
setAsAbsolutePath(context.asAbsolutePath.bind(context));
// Start loading the configs (asynchronously)
try { await loadConfigs() } catch (e) {
Logging.error`Could not load configs: ${e}`;
}
const manager = new Manager(context);
const manager = MANAGER = new Manager(context);
const subscribe = context.subscriptions.push.bind(context.subscriptions) as typeof context.subscriptions.push;
const registerCommand = (command: string, callback: (...args: any[]) => any, thisArg?: any) =>
@ -149,4 +147,8 @@ export async function activate(context: vscode.ExtensionContext) {
// sshfs.refresh()
registerCommand('sshfs.refresh', () => connectionTreeProvider.refresh());
subscribe(manager.connectionManager.onConnectionAdded(async con => {
await reloadWorkspaceFolderConfigs(con.actualConfig.name);
}));
}

@ -47,6 +47,7 @@ export class Manager implements vscode.TaskProvider, vscode.TerminalLinkProvider
});
}
public async createFileSystem(name: string, config?: FileSystemConfig): Promise<SSHFileSystem> {
await LOADING_CONFIGS; // Prevent race condition on startup, and wait for any current config reload to finish
const existing = this.fileSystems.find(fs => fs.authority === name);
if (existing) return existing;
let con: Connection | undefined;
@ -279,7 +280,7 @@ export class Manager implements vscode.TaskProvider, vscode.TerminalLinkProvider
return;
}
target = target.toLowerCase();
let configs = await loadConfigsRaw();
let configs = await loadConfigs();
configs = configs.filter(c => c.name === target);
if (configs.length === 0) {
vscode.window.showErrorMessage(`Found no matching configs for '${target}'`);

@ -4,7 +4,7 @@ import type { Message, Navigation } from 'common/webviewMessages';
import * as fs from 'fs';
import * as path from 'path';
import * as vscode from 'vscode';
import { deleteConfig, loadConfigsRaw, updateConfig } from './config';
import { deleteConfig, loadConfigs, updateConfig } from './config';
import { DEBUG, Logging as _Logging, LOGGING_NO_STACKTRACE } from './logging';
import { toPromise } from './utils';
@ -89,7 +89,7 @@ async function handleMessage(message: Message): Promise<any> {
}
switch (message.type) {
case 'requestData': {
const configs = await loadConfigsRaw();
const configs = await loadConfigs();
const locations = getLocations(configs);
return postMessage({
configs, locations,

Loading…
Cancel
Save