diff --git a/src/vs/base/node/encoding.ts b/src/vs/base/node/encoding.ts index 99eea2a9efdd0220abcaa341deed8be3ea8947d8..56aeddc9e5f5658192f4f0cde0ae2930d636444a 100644 --- a/src/vs/base/node/encoding.ts +++ b/src/vs/base/node/encoding.ts @@ -99,7 +99,12 @@ const MINIMUM_THRESHOLD = 0.2; //Todo. Decide how much this should be. const IGNORE_ENCODINGS = ['ascii', 'utf-8', 'utf-16', 'utf-32']; +function stripNonAlphaNumeric(encodingName: string): string { + return encodingName.replace(/[^a-zA-Z0-9]/g, '').toLowerCase(); +} + jschardet.Constants.MINIMUM_THRESHOLD = MINIMUM_THRESHOLD; + /** * Guesses the encoding from buffer. */ @@ -116,7 +121,7 @@ export function guessEncodingByBuffer(buffer: NodeBuffer): string { return null; } - return guessed.encoding; + return stripNonAlphaNumeric(guessed.encoding); } /** * The encodings that are allowed in a settings file don't match the canonical encoding labels specified by WHATWG. diff --git a/src/vs/base/node/mime.ts b/src/vs/base/node/mime.ts index e562fb0f1a56ffe6d1f0f57ac78bb7baa898ec39..a19515e05f384e3b4c3acfa13e53abdf3cef233d 100644 --- a/src/vs/base/node/mime.ts +++ b/src/vs/base/node/mime.ts @@ -58,19 +58,23 @@ export interface IMimeAndEncoding { mimes: string[]; } -function doDetectMimesFromStream(instream: streams.Readable, autoGuessEncoding: boolean): TPromise { +export interface DetectMimesOption { + autoGuessEncoding?: boolean; +} + +function doDetectMimesFromStream(instream: streams.Readable, option?: DetectMimesOption): TPromise { return stream.readExactlyByStream(instream, BUFFER_READ_MAX_LEN).then((readResult: stream.ReadResult) => { - return detectMimeAndEncodingFromBuffer(readResult, autoGuessEncoding); + return detectMimeAndEncodingFromBuffer(readResult, option && option.autoGuessEncoding); }); } -function doDetectMimesFromFile(absolutePath: string, autoGuessEncoding: boolean): TPromise { +function doDetectMimesFromFile(absolutePath: string, option?: DetectMimesOption): TPromise { return stream.readExactlyByFile(absolutePath, BUFFER_READ_MAX_LEN).then((readResult: stream.ReadResult) => { - return detectMimeAndEncodingFromBuffer(readResult, autoGuessEncoding); + return detectMimeAndEncodingFromBuffer(readResult, option && option.autoGuessEncoding); }); } -export function detectMimeAndEncodingFromBuffer({ buffer, bytesRead }: stream.ReadResult, autoGuessEncoding: boolean): IMimeAndEncoding { +export function detectMimeAndEncodingFromBuffer({ buffer, bytesRead }: stream.ReadResult, autoGuessEncoding?: boolean): IMimeAndEncoding { let enc = encoding.detectEncodingByBOMFromBuffer(buffer, bytesRead); // Detect 0 bytes to see if file is binary (ignore for UTF 16 though) @@ -123,8 +127,8 @@ function filterAndSortMimes(detectedMimes: string[], guessedMimes: string[]): st * @param instream the readable stream to detect the mime types from. * @param nameHint an additional hint that can be used to detect a mime from a file extension. */ -export function detectMimesFromStream(instream: streams.Readable, nameHint: string, autoGuessEncoding: boolean): TPromise { - return doDetectMimesFromStream(instream, autoGuessEncoding).then(encoding => +export function detectMimesFromStream(instream: streams.Readable, nameHint: string, option?: DetectMimesOption): TPromise { + return doDetectMimesFromStream(instream, option).then(encoding => handleMimeResult(nameHint, encoding) ); } @@ -133,8 +137,8 @@ export function detectMimesFromStream(instream: streams.Readable, nameHint: stri * Opens the given file to detect its mime type. Returns an array of mime types sorted from most specific to unspecific. * @param absolutePath the absolute path of the file. */ -export function detectMimesFromFile(absolutePath: string, autoGuessEncoding: boolean): TPromise { - return doDetectMimesFromFile(absolutePath, autoGuessEncoding).then(encoding => +export function detectMimesFromFile(absolutePath: string, option?: DetectMimesOption): TPromise { + return doDetectMimesFromFile(absolutePath, option).then(encoding => handleMimeResult(absolutePath, encoding) ); } diff --git a/src/vs/base/test/node/mime/mime.test.ts b/src/vs/base/test/node/mime/mime.test.ts index 92327daa9c168d127421b28ed78e7c2edaf97436..297cd62d137872b239713aeaf92b225be73e5254 100644 --- a/src/vs/base/test/node/mime/mime.test.ts +++ b/src/vs/base/test/node/mime/mime.test.ts @@ -14,7 +14,7 @@ suite('Mime', () => { test('detectMimesFromFile (JSON saved as PNG)', function (done: (err?: any) => void) { const file = require.toUrl('./fixtures/some.json.png'); - mime.detectMimesFromFile(file, false).then(mimes => { + mime.detectMimesFromFile(file).then(mimes => { assert.deepEqual(mimes.mimes, ['text/plain']); done(); }, done); @@ -23,7 +23,7 @@ suite('Mime', () => { test('detectMimesFromFile (PNG saved as TXT)', function (done: (err?: any) => void) { mimeCommon.registerTextMime({ id: 'text', mime: 'text/plain', extension: '.txt' }); const file = require.toUrl('./fixtures/some.png.txt'); - mime.detectMimesFromFile(file, false).then(mimes => { + mime.detectMimesFromFile(file).then(mimes => { assert.deepEqual(mimes.mimes, ['text/plain', 'application/octet-stream']); done(); }, done); @@ -31,7 +31,7 @@ suite('Mime', () => { test('detectMimesFromFile (XML saved as PNG)', function (done: (err?: any) => void) { const file = require.toUrl('./fixtures/some.xml.png'); - mime.detectMimesFromFile(file, false).then(mimes => { + mime.detectMimesFromFile(file).then(mimes => { assert.deepEqual(mimes.mimes, ['text/plain']); done(); }, done); @@ -39,7 +39,7 @@ suite('Mime', () => { test('detectMimesFromFile (QWOFF saved as TXT)', function (done: (err?: any) => void) { const file = require.toUrl('./fixtures/some.qwoff.txt'); - mime.detectMimesFromFile(file, false).then(mimes => { + mime.detectMimesFromFile(file).then(mimes => { assert.deepEqual(mimes.mimes, ['text/plain', 'application/octet-stream']); done(); }, done); @@ -47,7 +47,7 @@ suite('Mime', () => { test('detectMimesFromFile (CSS saved as QWOFF)', function (done: (err?: any) => void) { const file = require.toUrl('./fixtures/some.css.qwoff'); - mime.detectMimesFromFile(file, false).then(mimes => { + mime.detectMimesFromFile(file).then(mimes => { assert.deepEqual(mimes.mimes, ['text/plain']); done(); }, done); @@ -55,7 +55,7 @@ suite('Mime', () => { test('detectMimesFromFile (PDF)', function (done: () => void) { const file = require.toUrl('./fixtures/some.pdf'); - mime.detectMimesFromFile(file, false).then(mimes => { + mime.detectMimesFromFile(file).then(mimes => { assert.deepEqual(mimes.mimes, ['application/octet-stream']); done(); }, done); @@ -63,8 +63,8 @@ suite('Mime', () => { test('autoGuessEncoding (ShiftJIS)', function (done: () => void) { const file = require.toUrl('./fixtures/some.shiftjis.txt'); - mime.detectMimesFromFile(file, true).then(mimes => { - assert.equal(mimes.encoding, 'SHIFT_JIS'); + mime.detectMimesFromFile(file, { autoGuessEncoding: true }).then(mimes => { + assert.equal(mimes.encoding, 'shiftjis'); done(); }, done); }); diff --git a/src/vs/platform/files/common/files.ts b/src/vs/platform/files/common/files.ts index faaeaf1ddc4ad0310ed2007ec67e48cdf7c1be73..af54ea60f0ec0e9dc4f89361aee7d38863edde96 100644 --- a/src/vs/platform/files/common/files.ts +++ b/src/vs/platform/files/common/files.ts @@ -495,6 +495,11 @@ export interface IResolveContentOptions { * the contents of the file. */ encoding?: string; + + /** + * The optional guessEncoding parameter allows to guess encoding from content of the file. + */ + autoGuessEncoding?: boolean; } export interface IUpdateContentOptions { diff --git a/src/vs/workbench/browser/parts/editor/editorStatus.ts b/src/vs/workbench/browser/parts/editor/editorStatus.ts index b2b95a85d738ee591b2d76ed5788267408a917a9..35e35af75643d3e11950f753b9e4d0f96bf28eba 100644 --- a/src/vs/workbench/browser/parts/editor/editorStatus.ts +++ b/src/vs/workbench/browser/parts/editor/editorStatus.ts @@ -34,7 +34,7 @@ import { IEditor as IBaseEditor, IEditorInput } from 'vs/platform/editor/common/ import { IWorkbenchEditorService } from 'vs/workbench/services/editor/common/editorService'; import { IQuickOpenService, IPickOpenEntry, IFilePickOpenEntry } from 'vs/platform/quickOpen/common/quickOpen'; import { IWorkspaceConfigurationService } from 'vs/workbench/services/configuration/common/configuration'; -import { IFilesConfiguration, SUPPORTED_ENCODINGS } from 'vs/platform/files/common/files'; +import { IFilesConfiguration, SUPPORTED_ENCODINGS, IFileService } from 'vs/platform/files/common/files'; import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation'; import { IModeService } from 'vs/editor/common/services/modeService'; import { IModelService } from 'vs/editor/common/services/modelService'; @@ -1019,7 +1019,8 @@ export class ChangeEncodingAction extends Action { actionLabel: string, @IWorkbenchEditorService private editorService: IWorkbenchEditorService, @IQuickOpenService private quickOpenService: IQuickOpenService, - @IWorkspaceConfigurationService private configurationService: IWorkspaceConfigurationService + @IWorkspaceConfigurationService private configurationService: IWorkspaceConfigurationService, + @IFileService private fileService: IFileService ) { super(actionId, actionLabel); } @@ -1060,51 +1061,69 @@ export class ChangeEncodingAction extends Action { return undefined; } - return TPromise.timeout(50 /* quick open is sensitive to being opened so soon after another */).then(() => { - const configuration = this.configurationService.getConfiguration(); - - const isReopenWithEncoding = (action === reopenWithEncodingPick); - const configuredEncoding = configuration && configuration.files && configuration.files.encoding; - let directMatchIndex: number; - let aliasMatchIndex: number; - - // All encodings are valid picks - const picks: IPickOpenEntry[] = Object.keys(SUPPORTED_ENCODINGS) - .sort((k1, k2) => { - if (k1 === configuredEncoding) { - return -1; - } else if (k2 === configuredEncoding) { - return 1; - } - - return SUPPORTED_ENCODINGS[k1].order - SUPPORTED_ENCODINGS[k2].order; - }) - .filter(k => { - return !isReopenWithEncoding || !SUPPORTED_ENCODINGS[k].encodeOnly; // hide those that can only be used for encoding if we are about to decode - }) - .map((key, index) => { - if (key === encodingSupport.getEncoding()) { - directMatchIndex = index; - } else if (SUPPORTED_ENCODINGS[key].alias === encodingSupport.getEncoding()) { - aliasMatchIndex = index; - } + const guessEncoding = () => { + const uri = toResource(activeEditor.input); + return this.fileService.resolveContent(uri, { autoGuessEncoding: true }) + .then(content => content.encoding) + .then(encodingKey => encodingKey, err => null); + }; - return { id: key, label: SUPPORTED_ENCODINGS[key].labelLong }; - }); + return TPromise.timeout(50 /* quick open is sensitive to being opened so soon after another */) + .then(guessEncoding) + .then((guessedEncodingKey: string) => { + const configuration = this.configurationService.getConfiguration(); + + const isReopenWithEncoding = (action === reopenWithEncodingPick); + const configuredEncoding = configuration && configuration.files && configuration.files.encoding; + let directMatchIndex: number; + let aliasMatchIndex: number; + + // All encodings are valid picks + const picks: IPickOpenEntry[] = Object.keys(SUPPORTED_ENCODINGS) + .sort((k1, k2) => { + if (k1 === configuredEncoding) { + return -1; + } else if (k2 === configuredEncoding) { + return 1; + } + + return SUPPORTED_ENCODINGS[k1].order - SUPPORTED_ENCODINGS[k2].order; + }) + .filter(k => { + return !isReopenWithEncoding || !SUPPORTED_ENCODINGS[k].encodeOnly; // hide those that can only be used for encoding if we are about to decode + }) + .map((key, index) => { + if (key === encodingSupport.getEncoding()) { + directMatchIndex = index; + } else if (SUPPORTED_ENCODINGS[key].alias === encodingSupport.getEncoding()) { + aliasMatchIndex = index; + } + + return { id: key, label: SUPPORTED_ENCODINGS[key].labelLong }; + }); + + if (guessedEncodingKey && SUPPORTED_ENCODINGS[guessedEncodingKey]) { + const guessedLabel = nls.localize('pickEncodingLabelGuessed', "{0} (Guessed from content)"); + const guessedEncodingLabelLong = SUPPORTED_ENCODINGS[guessedEncodingKey].labelLong; + + picks[0].separator = { border: true }; + + picks.unshift({ id: guessedEncodingKey, label: strings.format(guessedLabel, guessedEncodingLabelLong) }); + } - return this.quickOpenService.pick(picks, { - placeHolder: isReopenWithEncoding ? nls.localize('pickEncodingForReopen', "Select File Encoding to Reopen File") : nls.localize('pickEncodingForSave', "Select File Encoding to Save with"), - autoFocus: { autoFocusIndex: typeof directMatchIndex === 'number' ? directMatchIndex : typeof aliasMatchIndex === 'number' ? aliasMatchIndex : void 0 } - }).then(encoding => { - if (encoding) { - activeEditor = this.editorService.getActiveEditor(); - encodingSupport = toEditorWithEncodingSupport(activeEditor.input); - if (encodingSupport && encodingSupport.getEncoding() !== encoding.id) { - encodingSupport.setEncoding(encoding.id, isReopenWithEncoding ? EncodingMode.Decode : EncodingMode.Encode); // Set new encoding + return this.quickOpenService.pick(picks, { + placeHolder: isReopenWithEncoding ? nls.localize('pickEncodingForReopen', "Select File Encoding to Reopen File") : nls.localize('pickEncodingForSave', "Select File Encoding to Save with"), + autoFocus: { autoFocusIndex: typeof directMatchIndex === 'number' ? directMatchIndex : typeof aliasMatchIndex === 'number' ? aliasMatchIndex : void 0 } + }).then(encoding => { + if (encoding) { + activeEditor = this.editorService.getActiveEditor(); + encodingSupport = toEditorWithEncodingSupport(activeEditor.input); + if (encodingSupport && encodingSupport.getEncoding() !== encoding.id) { + encodingSupport.setEncoding(encoding.id, isReopenWithEncoding ? EncodingMode.Decode : EncodingMode.Encode); // Set new encoding + } } - } + }); }); - }); }); } } diff --git a/src/vs/workbench/parts/git/node/git.lib.ts b/src/vs/workbench/parts/git/node/git.lib.ts index ff7981126d00d0ba3488957854c118260201baed..7fcfca449208ba70edd510efd1db0731953ac14e 100644 --- a/src/vs/workbench/parts/git/node/git.lib.ts +++ b/src/vs/workbench/parts/git/node/git.lib.ts @@ -327,7 +327,7 @@ export class Repository { return TPromise.wrapError(localize('errorBuffer', "Can't open file from git")); } - return detectMimesFromStream(child.stdout, null, false).then(result => { + return detectMimesFromStream(child.stdout, null).then(result => { return isBinaryMime(result.mimes) ? TPromise.wrapError({ message: localize('fileBinaryError', "File seems to be binary and cannot be opened as text"), diff --git a/src/vs/workbench/parts/git/node/rawGitService.ts b/src/vs/workbench/parts/git/node/rawGitService.ts index a633f05dd2ef80ae3c226c5fd94e2459d138fe54..1127fc6daefe53c1c70856b1a0f64d1d3e5f2b1a 100644 --- a/src/vs/workbench/parts/git/node/rawGitService.ts +++ b/src/vs/workbench/parts/git/node/rawGitService.ts @@ -165,14 +165,14 @@ export class RawGitService implements IRawGitService { detectMimetypes(filePath: string, treeish?: string): TPromise { return exists(join(this.repo.path, filePath)).then((exists) => { if (exists) { - return detectMimesFromFile(join(this.repo.path, filePath), false) + return detectMimesFromFile(join(this.repo.path, filePath)) .then(result => result.mimes); } const child = this.repo.show(treeish + ':' + filePath); return new TPromise((c, e) => - detectMimesFromStream(child.stdout, filePath, false) + detectMimesFromStream(child.stdout, filePath) .then(result => result.mimes) ); }); diff --git a/src/vs/workbench/services/files/node/fileService.ts b/src/vs/workbench/services/files/node/fileService.ts index 9718f499d0c36414f0d5ef19bd56601ff8d2955c..cb3a15031bea00ba3a52ea392a7e69acad8d2dec 100644 --- a/src/vs/workbench/services/files/node/fileService.ts +++ b/src/vs/workbench/services/files/node/fileService.ts @@ -206,7 +206,8 @@ export class FileService implements IFileService { } // 2.) detect mimes - return mime.detectMimesFromFile(absolutePath, this.options.autoGuessEncoding).then((detected: mime.IMimeAndEncoding) => { + const autoGuessEncoding = (options && options.autoGuessEncoding) || (this.options && this.options.autoGuessEncoding); + return mime.detectMimesFromFile(absolutePath, { autoGuessEncoding: autoGuessEncoding }).then((detected: mime.IMimeAndEncoding) => { const isText = detected.mimes.indexOf(baseMime.MIME_BINARY) === -1; // Return error early if client only accepts text and this is not text