Add TextSearchProvider (+ improve FileSearchProvider)

package.json

@@ -3,9 +3,10 @@
     "displayName": "SSH FS",
     "description": "File system provider using SSH",
     "publisher": "Kelvin",
-    "version": "1.14.7",
+    "version": "1.14.7-insiders",
+    "enableProposedApi": true,
     "engines": {
-        "vscode": "^1.26.0"
+        "vscode": "^1.33.0"
     },
     "categories": [
         "Other"
@@ -436,6 +437,7 @@
         "postinstall": "node ./node_modules/vscode/bin/install"
     },
     "devDependencies": {
+        "@types/minimatch": "^3.0.3",
         "@types/node": "^8.10.43",
         "@types/request": "^2.48.1",
         "@types/ssh2": "^0.5.35",
@@ -444,13 +446,14 @@
         "clean-webpack-plugin": "^2.0.0",
         "ts-loader": "^5.3.3",
         "typescript": "^3.3.3",
-        "vscode": "^1.1.21",
+        "vscode": "^1.1.33",
         "webpack": "^4.29.6",
         "webpack-cli": "^3.2.3"
     },
     "dependencies": {
         "event-stream": "^3.3.4",
         "jsonc-parser": "^2.0.0",
+        "minimatch": "^3.0.4",
         "socks": "^2.2.0",
         "ssh2": "^0.8.2",
         "winreg": "^1.2.4"

src/concurrencyLimiter.ts

@@ -0,0 +1,34 @@
+
+type Task = () => any;
+export class ConcurrencyLimiter {
+  protected tasks: Task[] = [];
+  protected workers: Promise<void>[] = [];
+  constructor(protected maxWorkers = 10) { }
+  public addTask(task: Task) {
+    this.tasks.push(task);
+    if (this.workers.length < this.maxWorkers) {
+      this.createWorker();
+    }
+  }
+  public clear() {
+    this.tasks = [];
+  }
+  public toPromise() {
+    if (!this.workers.length) return Promise.resolve();
+    return Promise.all(this.workers).then(() => this.toPromise());
+  }
+  protected async createWorker() {
+    const worker = this.taskRunner();
+    this.workers.push(worker);
+    worker.catch(() => { }).then(() => this.workers.splice(this.workers.indexOf(worker)));
+  }
+  protected async taskRunner() {
+    let task: Task | undefined = this.tasks.pop();
+    while (task) {
+      try {
+        await task();
+      } catch (e) { }
+      task = this.tasks.pop();
+    }
+  }
+}

src/extension.ts

@@ -4,6 +4,7 @@ import { loadConfigs } from './config';
 import { FileSystemConfig, invalidConfigName } from './fileSystemConfig';
 import * as Logging from './logging';
 import { Manager } from './manager';
+import { SearchProvider } from './searchProvider';
 
 function generateDetail(config: FileSystemConfig): string | undefined {
   const { username, host, putty } = config;
@@ -52,6 +53,9 @@ export function activate(context: vscode.ExtensionContext) {
     subscribe(vscode.commands.registerCommand(command, callback, thisArg));
 
   subscribe(vscode.workspace.registerFileSystemProvider('ssh', manager, { isCaseSensitive: true }));
+  const searchProvider = new SearchProvider(manager);
+  subscribe(vscode.workspace.registerFileSearchProvider('ssh', searchProvider));
+  subscribe(vscode.workspace.registerTextSearchProvider('ssh', searchProvider));
 
   async function pickAndClick(func: (name: string) => void, name?: string, activeOrNot?: boolean) {
     name = name || await pickConfig(manager, activeOrNot);

src/manager.ts

@@ -147,7 +147,7 @@ export class Manager implements vscode.FileSystemProvider, vscode.TreeDataProvid
         root = root.replace(/^~/, home.replace(/\/$/, ''));
       }
       const sftp = await getSFTP(client, config);
-      const fs = new SSHFileSystem(name, sftp, root, config!);
+      const fs = new SSHFileSystem(name, client, sftp, root, config!);
       Logging.info(`Created SSHFileSystem for ${name}, reading root directory...`);
       try {
         const rootUri = vscode.Uri.parse(`ssh://${name}/`);
@@ -311,5 +311,3 @@ export class Manager implements vscode.FileSystemProvider, vscode.TreeDataProvid
     return navigation ? navigate(navigation) : open();
   }
 }
-
-export default Manager;

src/searchProvider.ts

@@ -0,0 +1,160 @@
+
+import * as minimatch from 'minimatch';
+import { posix as path } from 'path';
+import * as vscode from 'vscode';
+import { ConcurrencyLimiter } from './concurrencyLimiter';
+import { Manager } from './manager';
+
+function textSearchQueryToRegExp(query: vscode.TextSearchQuery): RegExp {
+  const flags = `${query.isCaseSensitive ? '' : 'i'}mug`;
+  let pattern = query.pattern;
+  if (query.isRegExp) {
+    if (query.isWordMatch) pattern = `\\b${query.pattern}\\b`;
+    return new RegExp(pattern, flags);
+  }
+  pattern = pattern.replace(/\\/g, '\\\\');
+  if (query.isWordMatch) pattern = `\\b${query.pattern}\\b`;
+  return new RegExp(pattern, flags);
+}
+
+type RegExpMatchHandler = (lastIndex: number, mach: RegExpExecArray) => true | null | undefined | void;
+function forEachMatch(content: string, regex: RegExp, handler: RegExpMatchHandler) {
+  let mat = regex.exec(content);
+  while (mat) {
+    const { lastIndex } = regex;
+    if (handler(lastIndex, mat)) break;
+    mat = regex.exec(content);
+  }
+}
+
+type SimpleRange = [number, number];
+function getSearchResultRanges(content: string, query: vscode.TextSearchQuery, options: vscode.TextSearchOptions, regex: RegExp): SimpleRange[] {
+  const results: SimpleRange[] = [];
+  if (query.isMultiline) {
+    forEachMatch(content, /[^\r\n]+/g, (index, mat) => {
+      const res = getSearchResultRanges(mat[0], query, options, regex);
+      res.forEach(range => (range[0] += index, range[1] += index));
+      results.push(...res);
+    });
+    return results;
+  }
+  forEachMatch(content, regex, (index, mat) => {
+    results.push([index - mat[0].length, index]);
+  });
+  return results;
+}
+
+function getSearchResults(content: string, query: vscode.TextSearchQuery, options: vscode.TextSearchOptions, regexp: RegExp): vscode.Range[] {
+  const indexes = getSearchResultRanges(content, query, options, regexp);
+  const indexForLine: number[] = [0];
+  for (let i = 0; i < content.length; i += 1) {
+    if (content[i] === '\n') {
+      indexForLine.push(i + 1);
+    }
+  }
+  return indexes.map((range) => {
+    const startLine = indexForLine.findIndex(v => v >= range[0]) - 1;
+    const endLine = indexForLine.findIndex(v => v >= range[1]) - 1;
+    const startChar = range[0] - indexForLine[startLine];
+    const endChar = range[1] - indexForLine[endLine];
+    return new vscode.Range(startLine, startChar, endLine, endChar);
+  });
+}
+
+export class SearchProvider implements vscode.FileSearchProvider, vscode.TextSearchProvider {
+  protected cache: [vscode.CancellationToken, Promise<vscode.Uri[]>][] = [];
+  constructor(protected manager: Manager) { }
+  public async provideFileSearchResults(query: vscode.FileSearchQuery, options: vscode.FileSearchOptions, token: vscode.CancellationToken): Promise<vscode.Uri[]> {
+    const { folder, session } = options;
+    const paths = await this.getTree(options, session || token, !session);
+    if (token.isCancellationRequested) return [];
+    const pattern = query.pattern.toLowerCase();
+    return paths.map<vscode.Uri | false>((relative) => {
+      return relative.path.toLowerCase().includes(pattern) && relative;
+    }).filter(s => !!s) as vscode.Uri[];
+  }
+  public async provideTextSearchResults(query: vscode.TextSearchQuery, options: vscode.TextSearchOptions, progress: vscode.Progress<vscode.TextSearchResult>, token: vscode.CancellationToken): Promise<vscode.TextSearchComplete> {
+    const paths = await this.getTree(options, token, true);
+    const regexp = textSearchQueryToRegExp(query);
+    const limiter = new ConcurrencyLimiter(20);
+    token.onCancellationRequested(limiter.clear, limiter);
+    let found = 0;
+    const fs = (await this.manager.getFs(options.folder))!;
+    async function handleFile(uri: vscode.Uri) {
+      if (token.isCancellationRequested) return;
+      if (options.maxFileSize) {
+        const stats = await fs.stat(uri);
+        if (stats.size > options.maxFileSize) return;
+      }
+      const buffer = Buffer.from(await fs.readFile(uri));
+      const content = Buffer.from(buffer.toString(), options.encoding);
+      const ranges = getSearchResults(content.toString(), query, options, regexp);
+      found += getSearchResults.length;
+      progress.report({
+        ranges, uri,
+        preview: {
+          matches: ranges,
+          text: content.toString(),
+        },
+      } as vscode.TextSearchMatch);
+      if (found >= options.maxResults) limiter.clear();
+    }
+    for (const filepath of paths) {
+      limiter.addTask(() => handleFile(filepath));
+    }
+    await limiter.toPromise();
+    return { limitHit: found > options.maxResults };
+  }
+  protected async getTree(options: vscode.SearchOptions, session: vscode.CancellationToken, singleton = false): Promise<vscode.Uri[]> {
+    let cached = this.cache.find(([t]) => session === t);
+    if (cached) return await cached[1];
+    const singletonSource = singleton && new vscode.CancellationTokenSource();
+    if (singletonSource) {
+      session.onCancellationRequested(singletonSource.cancel, singletonSource);
+      singletonSource.token.onCancellationRequested(singletonSource.dispose, singletonSource);
+      session = singletonSource.token;
+    }
+    cached = [session, this.internal_buildTree(options, session)] as SearchProvider['cache'][0];
+    this.cache.push(cached);
+    session.onCancellationRequested(() => {
+      this.cache.splice(this.cache.indexOf(cached!));
+    });
+    const res = await cached[1];
+    if (singletonSource) singletonSource.cancel();
+    return res;
+  }
+  protected async internal_buildTree(options: vscode.FileSearchOptions, token: vscode.CancellationToken): Promise<vscode.Uri[]> {
+    // TODO: For the options, actually use the following: includes, useIgnoreFiles and useGlobalIgnoreFiles
+    const { folder } = options;
+    const fs = await this.manager.getFs(folder);
+    if (!fs || token.isCancellationRequested) return [];
+    const excludes = options.excludes.map(e => minimatch.makeRe(e, { nocase: true }));
+    const exclude = (p: string) => excludes.some(reg => reg.test(p));
+    const res: vscode.Uri[] = [];
+    async function readDirectory(uri: vscode.Uri) {
+      if (token.isCancellationRequested) return;
+      const entries = await fs!.readDirectory(uri).catch(() => [] as never);
+      if (token.isCancellationRequested) return;
+      return Promise.all(entries.map(([name, type]) => {
+        if (token.isCancellationRequested) return;
+        const joined = path.join(uri.path, name);
+        if (exclude(joined)) return;
+        // tslint:disable-next-line:no-bitwise
+        if (type & vscode.FileType.Directory) {
+          // tslint:disable-next-line:no-bitwise
+          if ((type & vscode.FileType.SymbolicLink) && !options.followSymlinks) return;
+          return readDirectory(uri.with({ path: joined }));
+        } else {
+          res.push(uri.with({ path: joined }));
+        }
+      }));
+    }
+    await readDirectory(folder);
+    if (options.includes.length) {
+      const includes = options.includes.map(e => minimatch.makeRe(e, { nocase: true }));
+      const include = (p: string) => includes.some(reg => reg.test(p));
+      return res.filter(uri => include(uri.path) || uri.path[0] === '/' && include(uri.path.slice(1)));
+    }
+    return res;
+  }
+}

src/sshFileSystem.ts

@@ -14,8 +14,11 @@ export class SSHFileSystem implements vscode.FileSystemProvider {
   public onDidChangeFile: vscode.Event<vscode.FileChangeEvent[]>;
   protected onDidChangeFileEmitter = new vscode.EventEmitter<vscode.FileChangeEvent[]>();
 
-  constructor(public readonly authority: string, protected sftp: ssh2.SFTPWrapper,
+  constructor(public readonly authority: string,
-              public readonly root: string, public readonly config: FileSystemConfig) {
+              public readonly client: ssh2.Client,
+              public readonly sftp: ssh2.SFTPWrapper,
+              public readonly root: string,
+              public readonly config: FileSystemConfig) {
     this.onDidChangeFile = this.onDidChangeFileEmitter.event;
     this.sftp.on('end', () => this.closed = true);
   }
@@ -25,10 +28,14 @@ export class SSHFileSystem implements vscode.FileSystemProvider {
     this.sftp.end();
   }
 
-  public relative(relPath: string) {
+  public absoluteFromRelative(relPath: string) {
     if (relPath.startsWith('/')) relPath = relPath.substr(1);
     return path.posix.resolve(this.root, relPath);
   }
+  public relativeFromAbsolute(absPath: string) {
+    if (!absPath.startsWith('/')) throw new Error('Not an absolute path');
+    return path.posix.relative(this.root, absPath);
+  }
 
   public continuePromise<T>(func: (cb: (err: Error | null, res?: T) => void) => boolean): Promise<T> {
     return new Promise<T>((resolve, reject) => {
@@ -55,7 +62,7 @@ export class SSHFileSystem implements vscode.FileSystemProvider {
     return new vscode.Disposable(() => { });
   }
   public async stat(uri: vscode.Uri): Promise<vscode.FileStat> {
-    const stat = await this.continuePromise<ssh2s.Stats>(cb => this.sftp.stat(this.relative(uri.path), cb)).catch((e: Error & { code: number }) => {
+    const stat = await this.continuePromise<ssh2s.Stats>(cb => this.sftp.stat(this.absoluteFromRelative(uri.path), cb)).catch((e: Error & { code: number }) => {
       throw e.code === 2 ? vscode.FileSystemError.FileNotFound(uri) : e;
     });
     const { mtime, size } = stat;
@@ -71,7 +78,7 @@ export class SSHFileSystem implements vscode.FileSystemProvider {
     };
   }
   public async readDirectory(uri: vscode.Uri): Promise<[string, vscode.FileType][]> {
-    const entries = await this.continuePromise<ssh2s.FileEntry[]>(cb => this.sftp.readdir(this.relative(uri.path), cb)).catch((e) => {
+    const entries = await this.continuePromise<ssh2s.FileEntry[]>(cb => this.sftp.readdir(this.absoluteFromRelative(uri.path), cb)).catch((e) => {
       throw e === 2 ? vscode.FileSystemError.FileNotFound(uri) : e;
     });
     return Promise.all(entries.map(async (file) => {
@@ -91,11 +98,11 @@ export class SSHFileSystem implements vscode.FileSystemProvider {
     }));
   }
   public createDirectory(uri: vscode.Uri): void | Promise<void> {
-    return this.continuePromise(cb => this.sftp.mkdir(this.relative(uri.path), cb));
+    return this.continuePromise(cb => this.sftp.mkdir(this.absoluteFromRelative(uri.path), cb));
   }
   public readFile(uri: vscode.Uri): Uint8Array | Promise<Uint8Array> {
     return new Promise((resolve, reject) => {
-      const stream = this.sftp.createReadStream(this.relative(uri.path), { autoClose: true });
+      const stream = this.sftp.createReadStream(this.absoluteFromRelative(uri.path), { autoClose: true });
       const bufs = [];
       stream.on('data', bufs.push.bind(bufs));
       stream.on('error', reject);
@@ -108,18 +115,18 @@ export class SSHFileSystem implements vscode.FileSystemProvider {
     return new Promise(async (resolve, reject) => {
       let mode: number | string | undefined;
       try {
-        const stat = await this.continuePromise<ssh2s.Stats>(cb => this.sftp.stat(this.relative(uri.path), cb));
+        const stat = await this.continuePromise<ssh2s.Stats>(cb => this.sftp.stat(this.absoluteFromRelative(uri.path), cb));
         mode = stat.mode;
       } catch (e) {
         if (e.message === 'No such file') {
           mode = this.config.newFileMode;
         } else {
           Logging.error(e);
-          vscode.window.showWarningMessage(`Couldn't read the permissions for '${this.relative(uri.path)}', permissions might be overwritten`);
+          vscode.window.showWarningMessage(`Couldn't read the permissions for '${this.absoluteFromRelative(uri.path)}', permissions might be overwritten`);
         }
       }
       mode = mode as number | undefined; // ssh2-streams supports an octal number as string, but ssh2's typings don't reflect this
-      const stream = this.sftp.createWriteStream(this.relative(uri.path), { mode, flags: 'w' });
+      const stream = this.sftp.createWriteStream(this.absoluteFromRelative(uri.path), { mode, flags: 'w' });
       stream.on('error', reject);
       stream.end(content, resolve);
     });
@@ -128,15 +135,15 @@ export class SSHFileSystem implements vscode.FileSystemProvider {
     const stats = await this.stat(uri);
     // tslint:disable no-bitwise */
     if (stats.type & (vscode.FileType.SymbolicLink | vscode.FileType.File)) {
-      return this.continuePromise(cb => this.sftp.unlink(this.relative(uri.path), cb));
+      return this.continuePromise(cb => this.sftp.unlink(this.absoluteFromRelative(uri.path), cb));
     } else if ((stats.type & vscode.FileType.Directory) && options.recursive) {
-      return this.continuePromise(cb => this.sftp.rmdir(this.relative(uri.path), cb));
+      return this.continuePromise(cb => this.sftp.rmdir(this.absoluteFromRelative(uri.path), cb));
     }
-    return this.continuePromise(cb => this.sftp.unlink(this.relative(uri.path), cb));
+    return this.continuePromise(cb => this.sftp.unlink(this.absoluteFromRelative(uri.path), cb));
     // tslint:enable no-bitwise */
   }
   public rename(oldUri: vscode.Uri, newUri: vscode.Uri, options: { overwrite: boolean; }): void | Promise<void> {
-    return this.continuePromise(cb => this.sftp.rename(this.relative(oldUri.path), this.relative(newUri.path), cb));
+    return this.continuePromise(cb => this.sftp.rename(this.absoluteFromRelative(oldUri.path), this.absoluteFromRelative(newUri.path), cb));
   }
 }