Add auto guessed entry to encoding picker

src/vs/base/node/encoding.ts

@@ -99,7 +99,12 @@ const MINIMUM_THRESHOLD = 0.2; //Todo. Decide how much this should be.
 
 const IGNORE_ENCODINGS = ['ascii', 'utf-8', 'utf-16', 'utf-32'];
 
+function stripNonAlphaNumeric(encodingName: string): string {
+	return encodingName.replace(/[^a-zA-Z0-9]/g, '').toLowerCase();
+}
+
 jschardet.Constants.MINIMUM_THRESHOLD = MINIMUM_THRESHOLD;
+
 /**
  * Guesses the encoding from buffer.
  */
@@ -116,7 +121,7 @@ export function guessEncodingByBuffer(buffer: NodeBuffer): string {
 		return null;
 	}
 
-	return guessed.encoding;
+	return stripNonAlphaNumeric(guessed.encoding);
 }
 /**
  * The encodings that are allowed in a settings file don't match the canonical encoding labels specified by WHATWG.

src/vs/base/node/mime.ts

@@ -58,19 +58,23 @@ export interface IMimeAndEncoding {
 	mimes: string[];
 }
 
-function doDetectMimesFromStream(instream: streams.Readable, autoGuessEncoding: boolean): TPromise<IMimeAndEncoding> {
+export interface DetectMimesOption {
+	autoGuessEncoding?: boolean;
+}
+
+function doDetectMimesFromStream(instream: streams.Readable, option?: DetectMimesOption): TPromise<IMimeAndEncoding> {
 	return stream.readExactlyByStream(instream, BUFFER_READ_MAX_LEN).then((readResult: stream.ReadResult) => {
-		return detectMimeAndEncodingFromBuffer(readResult, autoGuessEncoding);
+		return detectMimeAndEncodingFromBuffer(readResult, option && option.autoGuessEncoding);
 	});
 }
 
-function doDetectMimesFromFile(absolutePath: string, autoGuessEncoding: boolean): TPromise<IMimeAndEncoding> {
+function doDetectMimesFromFile(absolutePath: string, option?: DetectMimesOption): TPromise<IMimeAndEncoding> {
 	return stream.readExactlyByFile(absolutePath, BUFFER_READ_MAX_LEN).then((readResult: stream.ReadResult) => {
-		return detectMimeAndEncodingFromBuffer(readResult, autoGuessEncoding);
+		return detectMimeAndEncodingFromBuffer(readResult, option && option.autoGuessEncoding);
 	});
 }
 
-export function detectMimeAndEncodingFromBuffer({ buffer, bytesRead }: stream.ReadResult, autoGuessEncoding: boolean): IMimeAndEncoding {
+export function detectMimeAndEncodingFromBuffer({ buffer, bytesRead }: stream.ReadResult, autoGuessEncoding?: boolean): IMimeAndEncoding {
 	let enc = encoding.detectEncodingByBOMFromBuffer(buffer, bytesRead);
 
 	// Detect 0 bytes to see if file is binary (ignore for UTF 16 though)
@@ -123,8 +127,8 @@ function filterAndSortMimes(detectedMimes: string[], guessedMimes: string[]): st
  * @param instream the readable stream to detect the mime types from.
  * @param nameHint an additional hint that can be used to detect a mime from a file extension.
  */
-export function detectMimesFromStream(instream: streams.Readable, nameHint: string, autoGuessEncoding: boolean): TPromise<IMimeAndEncoding> {
-	return doDetectMimesFromStream(instream, autoGuessEncoding).then(encoding =>
+export function detectMimesFromStream(instream: streams.Readable, nameHint: string, option?: DetectMimesOption): TPromise<IMimeAndEncoding> {
+	return doDetectMimesFromStream(instream, option).then(encoding =>
 		handleMimeResult(nameHint, encoding)
 	);
 }
@@ -133,8 +137,8 @@ export function detectMimesFromStream(instream: streams.Readable, nameHint: stri
  * Opens the given file to detect its mime type. Returns an array of mime types sorted from most specific to unspecific.
  * @param absolutePath the absolute path of the file.
  */
-export function detectMimesFromFile(absolutePath: string, autoGuessEncoding: boolean): TPromise<IMimeAndEncoding> {
-	return doDetectMimesFromFile(absolutePath, autoGuessEncoding).then(encoding =>
+export function detectMimesFromFile(absolutePath: string, option?: DetectMimesOption): TPromise<IMimeAndEncoding> {
+	return doDetectMimesFromFile(absolutePath, option).then(encoding =>
 		handleMimeResult(absolutePath, encoding)
 	);
 }

src/vs/base/test/node/mime/mime.test.ts

@@ -14,7 +14,7 @@ suite('Mime', () => {
 
 	test('detectMimesFromFile (JSON saved as PNG)', function (done: (err?: any) => void) {
 		const file = require.toUrl('./fixtures/some.json.png');
-		mime.detectMimesFromFile(file, false).then(mimes => {
+		mime.detectMimesFromFile(file).then(mimes => {
 			assert.deepEqual(mimes.mimes, ['text/plain']);
 			done();
 		}, done);
@@ -23,7 +23,7 @@ suite('Mime', () => {
 	test('detectMimesFromFile (PNG saved as TXT)', function (done: (err?: any) => void) {
 		mimeCommon.registerTextMime({ id: 'text', mime: 'text/plain', extension: '.txt' });
 		const file = require.toUrl('./fixtures/some.png.txt');
-		mime.detectMimesFromFile(file, false).then(mimes => {
+		mime.detectMimesFromFile(file).then(mimes => {
 			assert.deepEqual(mimes.mimes, ['text/plain', 'application/octet-stream']);
 			done();
 		}, done);
@@ -31,7 +31,7 @@ suite('Mime', () => {
 
 	test('detectMimesFromFile (XML saved as PNG)', function (done: (err?: any) => void) {
 		const file = require.toUrl('./fixtures/some.xml.png');
-		mime.detectMimesFromFile(file, false).then(mimes => {
+		mime.detectMimesFromFile(file).then(mimes => {
 			assert.deepEqual(mimes.mimes, ['text/plain']);
 			done();
 		}, done);
@@ -39,7 +39,7 @@ suite('Mime', () => {
 
 	test('detectMimesFromFile (QWOFF saved as TXT)', function (done: (err?: any) => void) {
 		const file = require.toUrl('./fixtures/some.qwoff.txt');
-		mime.detectMimesFromFile(file, false).then(mimes => {
+		mime.detectMimesFromFile(file).then(mimes => {
 			assert.deepEqual(mimes.mimes, ['text/plain', 'application/octet-stream']);
 			done();
 		}, done);
@@ -47,7 +47,7 @@ suite('Mime', () => {
 
 	test('detectMimesFromFile (CSS saved as QWOFF)', function (done: (err?: any) => void) {
 		const file = require.toUrl('./fixtures/some.css.qwoff');
-		mime.detectMimesFromFile(file, false).then(mimes => {
+		mime.detectMimesFromFile(file).then(mimes => {
 			assert.deepEqual(mimes.mimes, ['text/plain']);
 			done();
 		}, done);
@@ -55,7 +55,7 @@ suite('Mime', () => {
 
 	test('detectMimesFromFile (PDF)', function (done: () => void) {
 		const file = require.toUrl('./fixtures/some.pdf');
-		mime.detectMimesFromFile(file, false).then(mimes => {
+		mime.detectMimesFromFile(file).then(mimes => {
 			assert.deepEqual(mimes.mimes, ['application/octet-stream']);
 			done();
 		}, done);
@@ -63,8 +63,8 @@ suite('Mime', () => {
 
 	test('autoGuessEncoding (ShiftJIS)', function (done: () => void) {
 		const file = require.toUrl('./fixtures/some.shiftjis.txt');
-		mime.detectMimesFromFile(file, true).then(mimes => {
-			assert.equal(mimes.encoding, 'SHIFT_JIS');
+		mime.detectMimesFromFile(file, { autoGuessEncoding: true }).then(mimes => {
+			assert.equal(mimes.encoding, 'shiftjis');
 			done();
 		}, done);
 	});

src/vs/platform/files/common/files.ts

@@ -495,6 +495,11 @@ export interface IResolveContentOptions {
 	 * the contents of the file.
 	 */
 	encoding?: string;
+
+	/**
+	 * The optional guessEncoding parameter allows to guess encoding from content of the file.
+	 */
+	autoGuessEncoding?: boolean;
 }
 
 export interface IUpdateContentOptions {

src/vs/workbench/browser/parts/editor/editorStatus.ts

@@ -34,7 +34,7 @@ import { IEditor as IBaseEditor, IEditorInput } from 'vs/platform/editor/common/
 import { IWorkbenchEditorService } from 'vs/workbench/services/editor/common/editorService';
 import { IQuickOpenService, IPickOpenEntry, IFilePickOpenEntry } from 'vs/platform/quickOpen/common/quickOpen';
 import { IWorkspaceConfigurationService } from 'vs/workbench/services/configuration/common/configuration';
-import { IFilesConfiguration, SUPPORTED_ENCODINGS } from 'vs/platform/files/common/files';
+import { IFilesConfiguration, SUPPORTED_ENCODINGS, IFileService } from 'vs/platform/files/common/files';
 import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
 import { IModeService } from 'vs/editor/common/services/modeService';
 import { IModelService } from 'vs/editor/common/services/modelService';
@@ -1019,7 +1019,8 @@ export class ChangeEncodingAction extends Action {
 		actionLabel: string,
 		@IWorkbenchEditorService private editorService: IWorkbenchEditorService,
 		@IQuickOpenService private quickOpenService: IQuickOpenService,
-		@IWorkspaceConfigurationService private configurationService: IWorkspaceConfigurationService
+		@IWorkspaceConfigurationService private configurationService: IWorkspaceConfigurationService,
+		@IFileService private fileService: IFileService
 	) {
 		super(actionId, actionLabel);
 	}
@@ -1060,51 +1061,69 @@ export class ChangeEncodingAction extends Action {
 				return undefined;
 			}
 
-			return TPromise.timeout(50 /* quick open is sensitive to being opened so soon after another */).then(() => {
-				const configuration = this.configurationService.getConfiguration<IFilesConfiguration>();
-
-				const isReopenWithEncoding = (action === reopenWithEncodingPick);
-				const configuredEncoding = configuration && configuration.files && configuration.files.encoding;
-				let directMatchIndex: number;
-				let aliasMatchIndex: number;
-
-				// All encodings are valid picks
-				const picks: IPickOpenEntry[] = Object.keys(SUPPORTED_ENCODINGS)
-					.sort((k1, k2) => {
-						if (k1 === configuredEncoding) {
-							return -1;
-						} else if (k2 === configuredEncoding) {
-							return 1;
-						}
-
-						return SUPPORTED_ENCODINGS[k1].order - SUPPORTED_ENCODINGS[k2].order;
-					})
-					.filter(k => {
-						return !isReopenWithEncoding || !SUPPORTED_ENCODINGS[k].encodeOnly; // hide those that can only be used for encoding if we are about to decode
-					})
-					.map((key, index) => {
-						if (key === encodingSupport.getEncoding()) {
-							directMatchIndex = index;
-						} else if (SUPPORTED_ENCODINGS[key].alias === encodingSupport.getEncoding()) {
-							aliasMatchIndex = index;
-						}
+			const guessEncoding = () => {
+				const uri = toResource(activeEditor.input);
+				return this.fileService.resolveContent(uri, { autoGuessEncoding: true })
+					.then(content => content.encoding)
+					.then(encodingKey => encodingKey, err => null);
+			};
 
-						return { id: key, label: SUPPORTED_ENCODINGS[key].labelLong };
-					});
+			return TPromise.timeout(50 /* quick open is sensitive to being opened so soon after another */)
+				.then(guessEncoding)
+				.then((guessedEncodingKey: string) => {
+					const configuration = this.configurationService.getConfiguration<IFilesConfiguration>();
+
+					const isReopenWithEncoding = (action === reopenWithEncodingPick);
+					const configuredEncoding = configuration && configuration.files && configuration.files.encoding;
+					let directMatchIndex: number;
+					let aliasMatchIndex: number;
+
+					// All encodings are valid picks
+					const picks: IPickOpenEntry[] = Object.keys(SUPPORTED_ENCODINGS)
+						.sort((k1, k2) => {
+							if (k1 === configuredEncoding) {
+								return -1;
+							} else if (k2 === configuredEncoding) {
+								return 1;
+							}
+
+							return SUPPORTED_ENCODINGS[k1].order - SUPPORTED_ENCODINGS[k2].order;
+						})
+						.filter(k => {
+							return !isReopenWithEncoding || !SUPPORTED_ENCODINGS[k].encodeOnly; // hide those that can only be used for encoding if we are about to decode
+						})
+						.map((key, index) => {
+							if (key === encodingSupport.getEncoding()) {
+								directMatchIndex = index;
+							} else if (SUPPORTED_ENCODINGS[key].alias === encodingSupport.getEncoding()) {
+								aliasMatchIndex = index;
+							}
+
+							return { id: key, label: SUPPORTED_ENCODINGS[key].labelLong };
+						});
+
+					if (guessedEncodingKey && SUPPORTED_ENCODINGS[guessedEncodingKey]) {
+						const guessedLabel = nls.localize('pickEncodingLabelGuessed', "{0}   (Guessed from content)");
+						const guessedEncodingLabelLong = SUPPORTED_ENCODINGS[guessedEncodingKey].labelLong;
+
+						picks[0].separator = { border: true };
+
+						picks.unshift({ id: guessedEncodingKey, label: strings.format(guessedLabel, guessedEncodingLabelLong) });
+					}
 
-				return this.quickOpenService.pick(picks, {
-					placeHolder: isReopenWithEncoding ? nls.localize('pickEncodingForReopen', "Select File Encoding to Reopen File") : nls.localize('pickEncodingForSave', "Select File Encoding to Save with"),
-					autoFocus: { autoFocusIndex: typeof directMatchIndex === 'number' ? directMatchIndex : typeof aliasMatchIndex === 'number' ? aliasMatchIndex : void 0 }
-				}).then(encoding => {
-					if (encoding) {
-						activeEditor = this.editorService.getActiveEditor();
-						encodingSupport = toEditorWithEncodingSupport(activeEditor.input);
-						if (encodingSupport && encodingSupport.getEncoding() !== encoding.id) {
-							encodingSupport.setEncoding(encoding.id, isReopenWithEncoding ? EncodingMode.Decode : EncodingMode.Encode); // Set new encoding
+					return this.quickOpenService.pick(picks, {
+						placeHolder: isReopenWithEncoding ? nls.localize('pickEncodingForReopen', "Select File Encoding to Reopen File") : nls.localize('pickEncodingForSave', "Select File Encoding to Save with"),
+						autoFocus: { autoFocusIndex: typeof directMatchIndex === 'number' ? directMatchIndex : typeof aliasMatchIndex === 'number' ? aliasMatchIndex : void 0 }
+					}).then(encoding => {
+						if (encoding) {
+							activeEditor = this.editorService.getActiveEditor();
+							encodingSupport = toEditorWithEncodingSupport(activeEditor.input);
+							if (encodingSupport && encodingSupport.getEncoding() !== encoding.id) {
+								encodingSupport.setEncoding(encoding.id, isReopenWithEncoding ? EncodingMode.Decode : EncodingMode.Encode); // Set new encoding
+							}
 						}
-					}
+					});
 				});
-			});
 		});
 	}
 }

src/vs/workbench/parts/git/node/git.lib.ts

@@ -327,7 +327,7 @@ export class Repository {
 			return TPromise.wrapError(localize('errorBuffer', "Can't open file from git"));
 		}
 
-		return detectMimesFromStream(child.stdout, null, false).then(result => {
+		return detectMimesFromStream(child.stdout, null).then(result => {
 			return isBinaryMime(result.mimes) ?
 				TPromise.wrapError<string>(<IFileOperationResult>{
 					message: localize('fileBinaryError', "File seems to be binary and cannot be opened as text"),

src/vs/workbench/parts/git/node/rawGitService.ts

@@ -165,14 +165,14 @@ export class RawGitService implements IRawGitService {
 	detectMimetypes(filePath: string, treeish?: string): TPromise<string[]> {
 		return exists(join(this.repo.path, filePath)).then((exists) => {
 			if (exists) {
-				return detectMimesFromFile(join(this.repo.path, filePath), false)
+				return detectMimesFromFile(join(this.repo.path, filePath))
 					.then(result => result.mimes);
 			}
 
 			const child = this.repo.show(treeish + ':' + filePath);
 
 			return new TPromise<string[]>((c, e) =>
-				detectMimesFromStream(child.stdout, filePath, false)
+				detectMimesFromStream(child.stdout, filePath)
 					.then(result => result.mimes)
 			);
 		});

src/vs/workbench/services/files/node/fileService.ts

@@ -206,7 +206,8 @@ export class FileService implements IFileService {
 			}
 
 			// 2.) detect mimes
-			return mime.detectMimesFromFile(absolutePath, this.options.autoGuessEncoding).then((detected: mime.IMimeAndEncoding) => {
+			const autoGuessEncoding = (options && options.autoGuessEncoding) || (this.options && this.options.autoGuessEncoding);
+			return mime.detectMimesFromFile(absolutePath, { autoGuessEncoding: autoGuessEncoding }).then((detected: mime.IMimeAndEncoding) => {
 				const isText = detected.mimes.indexOf(baseMime.MIME_BINARY) === -1;
 
 				// Return error early if client only accepts text and this is not text