From 534d10d37d302cc3b75d5c12021e4856826f07f5 Mon Sep 17 00:00:00 2001 From: Johannes Rieken Date: Mon, 9 Apr 2018 11:35:20 +0200 Subject: [PATCH] add FileSystemProvider2 for rapid changes, add readFile/writeFile, simplify delete, #47475 --- src/vs/platform/files/common/files.ts | 8 +- src/vs/vscode.proposed.d.ts | 40 ++++++- .../electron-browser/mainThreadFileSystem.ts | 42 ++----- src/vs/workbench/api/node/extHost.api.impl.ts | 4 +- src/vs/workbench/api/node/extHost.protocol.ts | 12 +- .../workbench/api/node/extHostFileSystem.ts | 112 ++++++++++++++---- .../electron-browser/remoteFileService.ts | 58 +++------ 7 files changed, 159 insertions(+), 117 deletions(-) diff --git a/src/vs/platform/files/common/files.ts b/src/vs/platform/files/common/files.ts index a6d2eac1d88..70d25fb09c6 100644 --- a/src/vs/platform/files/common/files.ts +++ b/src/vs/platform/files/common/files.ts @@ -12,7 +12,6 @@ import { isLinux } from 'vs/base/common/platform'; import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; import { Event } from 'vs/base/common/event'; import { startsWithIgnoreCase } from 'vs/base/common/strings'; -import { IProgress } from 'vs/platform/progress/common/progress'; import { IDisposable } from 'vs/base/common/lifecycle'; import { isEqualOrParent, isEqual } from 'vs/base/common/resources'; import { isUndefinedOrNull } from 'vs/base/common/types'; @@ -184,13 +183,12 @@ export interface IFileSystemProvider { // utimes(resource: URI, mtime: number, atime: number): TPromise; stat(resource: URI): TPromise; - read(resource: URI, offset: number, count: number, progress: IProgress): TPromise; - write(resource: URI, content: Uint8Array): TPromise; + readFile(resource: URI): TPromise; + writeFile(resource: URI, content: Uint8Array): TPromise; move(from: URI, to: URI): TPromise; mkdir(resource: URI): TPromise; readdir(resource: URI): TPromise<[URI, IStat][]>; - rmdir(resource: URI): TPromise; - unlink(resource: URI): TPromise; + delete(resource: URI): TPromise; } diff --git a/src/vs/vscode.proposed.d.ts b/src/vs/vscode.proposed.d.ts index 62e34f73e76..6f94afbe1ce 100644 --- a/src/vs/vscode.proposed.d.ts +++ b/src/vs/vscode.proposed.d.ts @@ -223,8 +223,46 @@ declare module 'vscode' { // create(resource: Uri): Thenable; } + // todo@joh discover files etc + // todo@joh CancellationToken everywhere + // todo@joh add open/close calls? + export interface FileSystemProvider2 { + + _version: 2; + + readonly onDidChange?: Event; + + // more... + // + utimes(resource: Uri, mtime: number, atime: number): Thenable; + + stat(resource: Uri): Thenable; + + readFile(resource: Uri, token: CancellationToken): Thenable; + + writeFile(resource: Uri, content: Uint8Array, token: CancellationToken): Thenable; + + // todo@remote + // Thenable + move(resource: Uri, target: Uri): Thenable; + + // todo@remote + // helps with performance bigly + // copy?(from: Uri, to: Uri): Thenable; + + + readdir(resource: Uri): Thenable<[Uri, FileStat][]>; + + // todo@remote + // ? useTrash, expose trash + delete(resource: Uri, options: { recursive?: boolean; }): Thenable; + + // todo@remote + create(resource: Uri, options: { type: FileType }): Thenable; + } + export namespace workspace { - export function registerFileSystemProvider(scheme: string, provider: FileSystemProvider): Disposable; + export function registerFileSystemProvider(scheme: string, provider: FileSystemProvider, newProvider?: FileSystemProvider2): Disposable; } //#endregion diff --git a/src/vs/workbench/api/electron-browser/mainThreadFileSystem.ts b/src/vs/workbench/api/electron-browser/mainThreadFileSystem.ts index 125bd331f22..ea33b46a78f 100644 --- a/src/vs/workbench/api/electron-browser/mainThreadFileSystem.ts +++ b/src/vs/workbench/api/electron-browser/mainThreadFileSystem.ts @@ -11,7 +11,6 @@ import { IFileService, IFileSystemProvider, IStat, IFileChange } from 'vs/platfo import { IDisposable, dispose } from 'vs/base/common/lifecycle'; import { Event, Emitter } from 'vs/base/common/event'; import { extHostNamedCustomer } from 'vs/workbench/api/electron-browser/extHostCustomers'; -import { IProgress } from 'vs/platform/progress/common/progress'; import { ISearchResultProvider, ISearchQuery, ISearchComplete, ISearchProgressItem, QueryType, IFileMatch, ISearchService, ILineMatch } from 'vs/platform/search/common/search'; import { values } from 'vs/base/common/map'; import { isFalsyOrEmpty } from 'vs/base/common/arrays'; @@ -55,11 +54,6 @@ export class MainThreadFileSystem implements MainThreadFileSystemShape { $onFileSystemChange(handle: number, changes: IFileChangeDto[]): void { this._fileProvider.get(handle).$onFileSystemChange(changes); } - - $reportFileChunk(handle: number, session: number, base64Chunk: string): void { - this._fileProvider.get(handle).reportFileChunk(session, base64Chunk); - } - // --- search $handleFindMatch(handle: number, session, data: UriComponents | [UriComponents, ILineMatch]): void { @@ -67,23 +61,10 @@ export class MainThreadFileSystem implements MainThreadFileSystemShape { } } -class FileReadOperation { - - private static _idPool = 0; - - constructor( - readonly progress: IProgress, - readonly id: number = ++FileReadOperation._idPool - ) { - // - } -} - class RemoteFileSystemProvider implements IFileSystemProvider { private readonly _onDidChange = new Emitter(); private readonly _registrations: IDisposable[]; - private readonly _reads = new Map(); readonly onDidChange: Event = this._onDidChange.event; @@ -118,25 +99,19 @@ class RemoteFileSystemProvider implements IFileSystemProvider { stat(resource: URI): TPromise { return this._proxy.$stat(this._handle, resource); } - read(resource: URI, offset: number, count: number, progress: IProgress): TPromise { - const read = new FileReadOperation(progress); - this._reads.set(read.id, read); - return this._proxy.$read(this._handle, read.id, offset, count, resource).then(value => { - this._reads.delete(read.id); - return value; + readFile(resource: URI): TPromise { + return this._proxy.$readFile(this._handle, resource).then(encoded => { + return Buffer.from(encoded, 'base64'); }); } - reportFileChunk(session: number, encodedChunk: string): void { - this._reads.get(session).progress.report(Buffer.from(encodedChunk, 'base64')); - } - write(resource: URI, content: Uint8Array): TPromise { + writeFile(resource: URI, content: Uint8Array): TPromise { let encoded = Buffer.isBuffer(content) ? content.toString('base64') : Buffer.from(content.buffer, content.byteOffset, content.byteLength).toString('base64'); - return this._proxy.$write(this._handle, resource, encoded); + return this._proxy.$writeFile(this._handle, resource, encoded); } - unlink(resource: URI): TPromise { - return this._proxy.$unlink(this._handle, resource); + delete(resource: URI): TPromise { + return this._proxy.$delete(this._handle, resource); } move(resource: URI, target: URI): TPromise { return this._proxy.$move(this._handle, resource, target); @@ -149,9 +124,6 @@ class RemoteFileSystemProvider implements IFileSystemProvider { return data.map(tuple => <[URI, IStat]>[URI.revive(tuple[0]), tuple[1]]); }); } - rmdir(resource: URI): TPromise { - return this._proxy.$rmdir(this._handle, resource); - } } class SearchOperation { diff --git a/src/vs/workbench/api/node/extHost.api.impl.ts b/src/vs/workbench/api/node/extHost.api.impl.ts index 34421925449..da7a17f7d25 100644 --- a/src/vs/workbench/api/node/extHost.api.impl.ts +++ b/src/vs/workbench/api/node/extHost.api.impl.ts @@ -538,8 +538,8 @@ export function createApiFactory( onDidEndTask: (listeners, thisArgs?, disposables?) => { return extHostTask.onDidEndTask(listeners, thisArgs, disposables); }, - registerFileSystemProvider: proposedApiFunction(extension, (scheme, provider) => { - return extHostFileSystem.registerFileSystemProvider(scheme, provider); + registerFileSystemProvider: proposedApiFunction(extension, (scheme, provider, newProvider?) => { + return extHostFileSystem.registerFileSystemProvider(scheme, provider, newProvider); }), registerSearchProvider: proposedApiFunction(extension, (scheme, provider) => { return extHostFileSystem.registerSearchProvider(scheme, provider); diff --git a/src/vs/workbench/api/node/extHost.protocol.ts b/src/vs/workbench/api/node/extHost.protocol.ts index 17a860bb3c3..7d29a27fb81 100644 --- a/src/vs/workbench/api/node/extHost.protocol.ts +++ b/src/vs/workbench/api/node/extHost.protocol.ts @@ -387,7 +387,6 @@ export interface MainThreadFileSystemShape extends IDisposable { $unregisterProvider(handle: number): void; $onFileSystemChange(handle: number, resource: IFileChangeDto[]): void; - $reportFileChunk(handle: number, session: number, base64Encoded: string | null): void; $handleFindMatch(handle: number, session, data: UriComponents | [UriComponents, ILineMatch]): void; } @@ -568,13 +567,16 @@ export interface ExtHostWorkspaceShape { export interface ExtHostFileSystemShape { $utimes(handle: number, resource: UriComponents, mtime: number, atime: number): TPromise; $stat(handle: number, resource: UriComponents): TPromise; - $read(handle: number, session: number, offset: number, count: number, resource: UriComponents): TPromise; - $write(handle: number, resource: UriComponents, base64Encoded: string): TPromise; - $unlink(handle: number, resource: UriComponents): TPromise; + + $readFile(handle: number, resource: UriComponents): TPromise; + $writeFile(handle: number, resource: UriComponents, base64Encoded: string): TPromise; + $move(handle: number, resource: UriComponents, target: UriComponents): TPromise; $mkdir(handle: number, resource: UriComponents): TPromise; $readdir(handle: number, resource: UriComponents): TPromise<[UriComponents, IStat][]>; - $rmdir(handle: number, resource: UriComponents): TPromise; + + $delete(handle: number, resource: UriComponents): TPromise; + $provideFileSearchResults(handle: number, session: number, query: string): TPromise; $provideTextSearchResults(handle: number, session: number, pattern: IPatternInfo, options: { includes: string[], excludes: string[] }): TPromise; } diff --git a/src/vs/workbench/api/node/extHostFileSystem.ts b/src/vs/workbench/api/node/extHostFileSystem.ts index bf7def2343f..db8749594af 100644 --- a/src/vs/workbench/api/node/extHostFileSystem.ts +++ b/src/vs/workbench/api/node/extHostFileSystem.ts @@ -13,9 +13,8 @@ import { IDisposable } from 'vs/base/common/lifecycle'; import { asWinJsPromise } from 'vs/base/common/async'; import { IPatternInfo } from 'vs/platform/search/common/search'; import { values } from 'vs/base/common/map'; -import { Range } from 'vs/workbench/api/node/extHostTypes'; +import { Range, FileType } from 'vs/workbench/api/node/extHostTypes'; import { ExtHostLanguageFeatures } from 'vs/workbench/api/node/extHostLanguageFeatures'; -import { IProgress } from 'vs/platform/progress/common/progress'; class FsLinkProvider implements vscode.DocumentLinkProvider { @@ -56,10 +55,70 @@ class FsLinkProvider implements vscode.DocumentLinkProvider { } } + +class FileSystemProviderShim implements vscode.FileSystemProvider2 { + + _version: 2; + + onDidChange?: vscode.Event; + + constructor(private readonly _delegate: vscode.FileSystemProvider) { + this.onDidChange = this._delegate.onDidChange; + } + + utimes(resource: vscode.Uri, mtime: number, atime: number): Thenable { + return this._delegate.utimes(resource, mtime, atime); + } + stat(resource: vscode.Uri): Thenable { + return this._delegate.stat(resource); + } + move(resource: vscode.Uri, target: vscode.Uri): Thenable { + return this.move(resource, target); + } + readdir(resource: vscode.Uri): Thenable<[vscode.Uri, vscode.FileStat][]> { + return this._delegate.readdir(resource); + } + + // --- delete/create file or folder + + delete(resource: vscode.Uri, options: { recursive: boolean; }): Thenable { + return this.stat(resource).then(stat => { + if (stat.type === FileType.Dir) { + return this._delegate.rmdir(resource); + } else { + return this._delegate.unlink(resource); + } + }); + } + create(resource: vscode.Uri, options: { type: vscode.FileType; }): Thenable { + if (options.type === FileType.Dir) { + return this._delegate.mkdir(resource); + } else { + return this._delegate.write(resource, Buffer.from([])).then(() => this._delegate.stat(resource)); + } + } + + // --- read/write + + readFile(resource: vscode.Uri): Thenable { + let chunks: Buffer[] = []; + return this._delegate.read(resource, 0, -1, { + report(data) { + chunks.push(Buffer.from(data)); + } + }).then(() => { + return Buffer.concat(chunks); + }); + } + writeFile(resource: vscode.Uri, content: Uint8Array): Thenable { + return this._delegate.write(resource, content); + } +} + export class ExtHostFileSystem implements ExtHostFileSystemShape { private readonly _proxy: MainThreadFileSystemShape; - private readonly _fsProvider = new Map(); + private readonly _fsProvider = new Map(); private readonly _searchProvider = new Map(); private readonly _linkProvider = new FsLinkProvider(); @@ -70,7 +129,15 @@ export class ExtHostFileSystem implements ExtHostFileSystemShape { extHostLanguageFeatures.registerDocumentLinkProvider('*', this._linkProvider); } - registerFileSystemProvider(scheme: string, provider: vscode.FileSystemProvider) { + registerFileSystemProvider(scheme: string, provider: vscode.FileSystemProvider, newProvider: vscode.FileSystemProvider2) { + if (newProvider && newProvider._version === 2) { + return this._doRegisterFileSystemProvider(scheme, newProvider); + } else { + return this._doRegisterFileSystemProvider(scheme, new FileSystemProviderShim(provider)); + } + } + + private _doRegisterFileSystemProvider(scheme: string, provider: vscode.FileSystemProvider2) { const handle = this._handlePool++; this._linkProvider.add(scheme); this._fsProvider.set(handle, provider); @@ -109,36 +176,29 @@ export class ExtHostFileSystem implements ExtHostFileSystemShape { $stat(handle: number, resource: UriComponents): TPromise { return asWinJsPromise(token => this._fsProvider.get(handle).stat(URI.revive(resource))); } - $read(handle: number, session: number, offset: number, count: number, resource: UriComponents): TPromise { - const progress: IProgress = { - report: chunk => { - let base64Chunk = Buffer.isBuffer(chunk) - ? chunk.toString('base64') - : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength).toString('base64'); - - this._proxy.$reportFileChunk(handle, session, base64Chunk); - } - }; - return asWinJsPromise(token => this._fsProvider.get(handle).read(URI.revive(resource), offset, count, progress)); + $readdir(handle: number, resource: UriComponents): TPromise<[UriComponents, IStat][], any> { + return asWinJsPromise(token => this._fsProvider.get(handle).readdir(URI.revive(resource))); } - $write(handle: number, resource: UriComponents, base64Content: string): TPromise { - return asWinJsPromise(token => this._fsProvider.get(handle).write(URI.revive(resource), Buffer.from(base64Content, 'base64'))); + $readFile(handle: number, resource: UriComponents): TPromise { + return asWinJsPromise(token => { + return this._fsProvider.get(handle).readFile(URI.revive(resource), token); + }).then(data => { + return Buffer.isBuffer(data) ? data.toString('base64') : Buffer.from(data.buffer, data.byteOffset, data.byteLength).toString('base64'); + }); } - $unlink(handle: number, resource: UriComponents): TPromise { - return asWinJsPromise(token => this._fsProvider.get(handle).unlink(URI.revive(resource))); + $writeFile(handle: number, resource: UriComponents, base64Content: string): TPromise { + return asWinJsPromise(token => this._fsProvider.get(handle).writeFile(URI.revive(resource), Buffer.from(base64Content, 'base64'), token)); + } + $delete(handle: number, resource: UriComponents): TPromise { + return asWinJsPromise(token => this._fsProvider.get(handle).delete(URI.revive(resource), { recursive: true })); } $move(handle: number, resource: UriComponents, target: UriComponents): TPromise { return asWinJsPromise(token => this._fsProvider.get(handle).move(URI.revive(resource), URI.revive(target))); } $mkdir(handle: number, resource: UriComponents): TPromise { - return asWinJsPromise(token => this._fsProvider.get(handle).mkdir(URI.revive(resource))); - } - $readdir(handle: number, resource: UriComponents): TPromise<[UriComponents, IStat][], any> { - return asWinJsPromise(token => this._fsProvider.get(handle).readdir(URI.revive(resource))); - } - $rmdir(handle: number, resource: UriComponents): TPromise { - return asWinJsPromise(token => this._fsProvider.get(handle).rmdir(URI.revive(resource))); + return asWinJsPromise(token => this._fsProvider.get(handle).create(URI.revive(resource), { type: FileType.Dir })); } + $provideFileSearchResults(handle: number, session: number, query: string): TPromise { const provider = this._searchProvider.get(handle); if (!provider.provideFileSearchResults) { diff --git a/src/vs/workbench/services/files/electron-browser/remoteFileService.ts b/src/vs/workbench/services/files/electron-browser/remoteFileService.ts index 8ea31395c81..9d283ff9690 100644 --- a/src/vs/workbench/services/files/electron-browser/remoteFileService.ts +++ b/src/vs/workbench/services/files/electron-browser/remoteFileService.ts @@ -12,7 +12,6 @@ import { posix } from 'path'; import { IDisposable } from 'vs/base/common/lifecycle'; import { isFalsyOrEmpty, distinct } from 'vs/base/common/arrays'; import { Schemas } from 'vs/base/common/network'; -import { Progress } from 'vs/platform/progress/common/progress'; import { decodeStream, encode, UTF8, UTF8_with_bom, detectEncodingFromBuffer, maxEncodingDetectionBufferLen } from 'vs/base/node/encoding'; import { TernarySearchTree } from 'vs/base/common/map'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; @@ -213,7 +212,7 @@ export class RemoteFileService extends FileService { if (resource.scheme === Schemas.file) { return super.resolveContent(resource, options); } else { - return this._doResolveContent(resource, options).then(RemoteFileService._asContent); + return this._readFile(resource, options).then(RemoteFileService._asContent); } } @@ -221,11 +220,11 @@ export class RemoteFileService extends FileService { if (resource.scheme === Schemas.file) { return super.resolveStreamContent(resource, options); } else { - return this._doResolveContent(resource, options); + return this._readFile(resource, options); } } - private _doResolveContent(resource: URI, options: IResolveContentOptions = Object.create(null)): TPromise { + private _readFile(resource: URI, options: IResolveContentOptions = Object.create(null)): TPromise { return this._withProvider(resource).then(provider => { return this.resolveFile(resource).then(fileStat => { @@ -249,15 +248,11 @@ export class RemoteFileService extends FileService { const guessEncoding = options.autoGuessEncoding; const count = maxEncodingDetectionBufferLen(options); - const chunks: Buffer[] = []; - - return provider.read( - resource, - 0, count, - new Progress(chunk => chunks.push(chunk)) - ).then(bytesRead => { - return detectEncodingFromBuffer({ bytesRead, buffer: Buffer.concat(chunks) }, guessEncoding); + let buffer: Buffer; + return provider.readFile(resource).then(data => { + buffer = Buffer.from(data); + return detectEncodingFromBuffer({ bytesRead: Math.min(count, buffer.length), buffer }, guessEncoding); }).then(detected => { if (options.acceptTextOnly && detected.seemsBinary) { return TPromise.wrapError(new FileOperationError( @@ -289,27 +284,7 @@ export class RemoteFileService extends FileService { // const encoding = this.getEncoding(resource); const stream = decodeStream(preferredEncoding); - - // start with what we have already read - // and have a new stream to read the rest - let offset = 0; - for (const chunk of chunks) { - stream.write(chunk); - offset += chunk.length; - } - if (offset < count) { - // we didn't read enough the first time which means - // that we are done - stream.end(); - } else { - // there is more to read - provider.read(resource, offset, -1, new Progress(chunk => stream.write(chunk))).then(() => { - stream.end(); - }, err => { - stream.emit('error', err); - stream.end(); - }); - } + stream.end(buffer); return { encoding: preferredEncoding, @@ -340,7 +315,7 @@ export class RemoteFileService extends FileService { if (exists && options && !options.overwrite) { return TPromise.wrapError(new FileOperationError('EEXIST', FileOperationResult.FILE_MODIFIED_SINCE, options)); } - return this._doUpdateContent(provider, resource, content || '', {}); + return this._writeFile(provider, resource, content || '', {}); }).then(fileStat => { this._onAfterOperation.fire(new FileOperationEvent(resource, FileOperation.CREATE, fileStat)); return fileStat; @@ -354,15 +329,15 @@ export class RemoteFileService extends FileService { return super.updateContent(resource, value, options); } else { return this._withProvider(resource).then(provider => { - return this._doUpdateContent(provider, resource, value, options || {}); + return this._writeFile(provider, resource, value, options || {}); }); } } - private _doUpdateContent(provider: IFileSystemProvider, resource: URI, content: string | ITextSnapshot, options: IUpdateContentOptions): TPromise { + private _writeFile(provider: IFileSystemProvider, resource: URI, content: string | ITextSnapshot, options: IUpdateContentOptions): TPromise { const encoding = this.getEncoding(resource, options.encoding); // TODO@Joh support streaming API for remote file system writes - return provider.write(resource, encode(typeof content === 'string' ? content : snapshotToString(content), encoding)).then(() => { + return provider.writeFile(resource, encode(typeof content === 'string' ? content : snapshotToString(content), encoding)).then(() => { return this.resolveFile(resource); }); } @@ -390,9 +365,7 @@ export class RemoteFileService extends FileService { return super.del(resource, useTrash); } else { return this._withProvider(resource).then(provider => { - return provider.stat(resource).then(stat => { - return stat.type === FileType.Dir ? provider.rmdir(resource) : provider.unlink(resource); - }).then(() => { + return provider.delete(resource).then(() => { this._onAfterOperation.fire(new FileOperationEvent(resource, FileOperation.DELETE)); }); }); @@ -483,7 +456,7 @@ export class RemoteFileService extends FileService { // https://github.com/Microsoft/vscode/issues/41543 return this.resolveContent(source, { acceptTextOnly: true }).then(content => { return this._withProvider(target).then(provider => { - return this._doUpdateContent(provider, target, content.value, { encoding: content.encoding }).then(fileStat => { + return this._writeFile(provider, target, content.value, { encoding: content.encoding }).then(fileStat => { this._onAfterOperation.fire(new FileOperationEvent(source, FileOperation.COPY, fileStat)); return fileStat; }); @@ -497,7 +470,6 @@ export class RemoteFileService extends FileService { }); }); }); - } touchFile(resource: URI): TPromise { @@ -513,7 +485,7 @@ export class RemoteFileService extends FileService { return provider.stat(resource).then(() => { return provider.utimes(resource, Date.now(), Date.now()); }, err => { - return provider.write(resource, new Uint8Array(0)); + return provider.writeFile(resource, new Uint8Array(0)); }).then(() => { return this.resolveFile(resource); }); -- GitLab