Add flag to decide whether to automatically detect

src/vs/base/node/mime.ts

@@ -58,15 +58,19 @@ export interface IMimeAndEncoding {
 	mimes: string[];
 }
 
-function doDetectMimesFromStream(instream: streams.Readable): TPromise<IMimeAndEncoding> {
-	return stream.readExactlyByStream(instream, BUFFER_READ_MAX_LEN).then(detectMimeAndEncodingFromBuffer);
+function doDetectMimesFromStream(instream: streams.Readable, autoDetectEncoding: boolean): TPromise<IMimeAndEncoding> {
+	return stream.readExactlyByStream(instream, BUFFER_READ_MAX_LEN).then((readResult: stream.ReadResult) => {
+		return detectMimeAndEncodingFromBuffer(readResult, autoDetectEncoding);
+	});
 }
 
-function doDetectMimesFromFile(absolutePath: string): TPromise<IMimeAndEncoding> {
-	return stream.readExactlyByFile(absolutePath, BUFFER_READ_MAX_LEN).then(detectMimeAndEncodingFromBuffer);
+function doDetectMimesFromFile(absolutePath: string, autoDetectEncoding: boolean): TPromise<IMimeAndEncoding> {
+	return stream.readExactlyByFile(absolutePath, BUFFER_READ_MAX_LEN).then((readResult: stream.ReadResult) => {
+		return detectMimeAndEncodingFromBuffer(readResult, autoDetectEncoding);
+	});
 }
 
-export function detectMimeAndEncodingFromBuffer({buffer, bytesRead}: stream.ReadResult): IMimeAndEncoding {
+export function detectMimeAndEncodingFromBuffer({buffer, bytesRead}: stream.ReadResult, autoDetectEncoding: boolean): IMimeAndEncoding {
 	let enc = encoding.detectEncodingByBOMFromBuffer(buffer, bytesRead);
 
 	// Detect 0 bytes to see if file is binary (ignore for UTF 16 though)
@@ -79,7 +83,7 @@ export function detectMimeAndEncodingFromBuffer({buffer, bytesRead}: stream.Read
 			}
 		}
 	}
-	if (isText && !enc) {
+	if (autoDetectEncoding && isText && !enc) {
 		enc = encoding.detectEncodingByBuffer(buffer);
 	}
 
@@ -119,8 +123,8 @@ function filterAndSortMimes(detectedMimes: string[], guessedMimes: string[]): st
  * @param instream the readable stream to detect the mime types from.
  * @param nameHint an additional hint that can be used to detect a mime from a file extension.
  */
-export function detectMimesFromStream(instream: streams.Readable, nameHint: string): TPromise<IMimeAndEncoding> {
-	return doDetectMimesFromStream(instream).then(encoding =>
+export function detectMimesFromStream(instream: streams.Readable, nameHint: string, autoDetectEncoding: boolean): TPromise<IMimeAndEncoding> {
+	return doDetectMimesFromStream(instream, autoDetectEncoding).then(encoding =>
 		handleMimeResult(nameHint, encoding)
 	);
 }
@@ -129,8 +133,8 @@ export function detectMimesFromStream(instream: streams.Readable, nameHint: stri
  * Opens the given file to detect its mime type. Returns an array of mime types sorted from most specific to unspecific.
  * @param absolutePath the absolute path of the file.
  */
-export function detectMimesFromFile(absolutePath: string): TPromise<IMimeAndEncoding> {
-	return doDetectMimesFromFile(absolutePath).then(encoding =>
+export function detectMimesFromFile(absolutePath: string, autoDetectEncoding: boolean): TPromise<IMimeAndEncoding> {
+	return doDetectMimesFromFile(absolutePath, autoDetectEncoding).then(encoding =>
 		handleMimeResult(absolutePath, encoding)
 	);
 }

src/vs/base/test/node/mime/fixtures/some.shiftjis.txt

+VSCODE͍ō̃GfB^B
\ No newline at end of file

src/vs/base/test/node/mime/mime.test.ts

@@ -14,7 +14,7 @@ suite('Mime', () => {
 
 	test('detectMimesFromFile (JSON saved as PNG)', function (done: (err?: any) => void) {
 		const file = require.toUrl('./fixtures/some.json.png');
-		mime.detectMimesFromFile(file).then(mimes => {
+		mime.detectMimesFromFile(file, false).then(mimes => {
 			assert.deepEqual(mimes.mimes, ['text/plain']);
 			done();
 		}, done);
@@ -23,7 +23,7 @@ suite('Mime', () => {
 	test('detectMimesFromFile (PNG saved as TXT)', function (done: (err?: any) => void) {
 		mimeCommon.registerTextMime({ id: 'text', mime: 'text/plain', extension: '.txt' });
 		const file = require.toUrl('./fixtures/some.png.txt');
-		mime.detectMimesFromFile(file).then(mimes => {
+		mime.detectMimesFromFile(file, false).then(mimes => {
 			assert.deepEqual(mimes.mimes, ['text/plain', 'application/octet-stream']);
 			done();
 		}, done);
@@ -31,7 +31,7 @@ suite('Mime', () => {
 
 	test('detectMimesFromFile (XML saved as PNG)', function (done: (err?: any) => void) {
 		const file = require.toUrl('./fixtures/some.xml.png');
-		mime.detectMimesFromFile(file).then(mimes => {
+		mime.detectMimesFromFile(file, false).then(mimes => {
 			assert.deepEqual(mimes.mimes, ['text/plain']);
 			done();
 		}, done);
@@ -39,7 +39,7 @@ suite('Mime', () => {
 
 	test('detectMimesFromFile (QWOFF saved as TXT)', function (done: (err?: any) => void) {
 		const file = require.toUrl('./fixtures/some.qwoff.txt');
-		mime.detectMimesFromFile(file).then(mimes => {
+		mime.detectMimesFromFile(file, false).then(mimes => {
 			assert.deepEqual(mimes.mimes, ['text/plain', 'application/octet-stream']);
 			done();
 		}, done);
@@ -47,7 +47,7 @@ suite('Mime', () => {
 
 	test('detectMimesFromFile (CSS saved as QWOFF)', function (done: (err?: any) => void) {
 		const file = require.toUrl('./fixtures/some.css.qwoff');
-		mime.detectMimesFromFile(file).then(mimes => {
+		mime.detectMimesFromFile(file, false).then(mimes => {
 			assert.deepEqual(mimes.mimes, ['text/plain']);
 			done();
 		}, done);
@@ -55,9 +55,17 @@ suite('Mime', () => {
 
 	test('detectMimesFromFile (PDF)', function (done: () => void) {
 		const file = require.toUrl('./fixtures/some.pdf');
-		mime.detectMimesFromFile(file).then(mimes => {
+		mime.detectMimesFromFile(file, false).then(mimes => {
 			assert.deepEqual(mimes.mimes, ['application/octet-stream']);
 			done();
 		}, done);
 	});
+
+	test('autoDetectEncoding (ShiftJIS)', function (done: () => void) {
+		const file = require.toUrl('./fixtures/some.shiftjis.txt');
+		mime.detectMimesFromFile(file, true).then(mimes => {
+			assert.deepEqual(mimes.encoding, 'SHIFT_JIS');
+			done();
+		}, done);
+	});
 });

src/vs/platform/files/common/files.ts

@@ -575,6 +575,7 @@ export interface IFilesConfiguration {
 		exclude: glob.IExpression;
 		watcherExclude: { [filepattern: string]: boolean };
 		encoding: string;
+		autoDetectEncoding: boolean;
 		trimTrailingWhitespace: boolean;
 		autoSave: string;
 		autoSaveDelay: number;

src/vs/platform/telemetry/common/telemetryUtils.ts

@@ -240,6 +240,7 @@ const configurationValueWhitelist = [
 	'editor.acceptSuggestionOnCommitCharacter',
 	'workbench.editor.showTabs',
 	'files.encoding',
+	'files.autoDetectEncoding',
 	'editor.quickSuggestionsDelay',
 	'editor.snippetSuggestions',
 	'editor.selectionHighlight',

src/vs/workbench/parts/files/browser/files.contribution.ts

@@ -212,6 +212,11 @@ configurationRegistry.registerConfiguration({
 			'default': 'utf8',
 			'description': nls.localize('encoding', "The default character set encoding to use when reading and writing files."),
 		},
+		'files.autoDetectEncoding': {
+			'type': 'boolean',
+			'default': false,
+			'description': nls.localize('autoDetetEncoding', "whem enabled, will use detected encoding in opening a text file.")
+		},
 		'files.eol': {
 			'type': 'string',
 			'enum': [

src/vs/workbench/parts/git/node/git.lib.ts

@@ -327,7 +327,7 @@ export class Repository {
 			return TPromise.wrapError(localize('errorBuffer', "Can't open file from git"));
 		}
 
-		return detectMimesFromStream(child.stdout, null).then(result => {
+		return detectMimesFromStream(child.stdout, null, false).then(result => {
 			return isBinaryMime(result.mimes) ?
 				TPromise.wrapError<string>(<IFileOperationResult>{
 					message: localize('fileBinaryError', "File seems to be binary and cannot be opened as text"),

src/vs/workbench/parts/git/node/rawGitService.ts

@@ -165,14 +165,14 @@ export class RawGitService implements IRawGitService {
 	detectMimetypes(filePath: string, treeish?: string): TPromise<string[]> {
 		return exists(join(this.repo.path, filePath)).then((exists) => {
 			if (exists) {
-				return detectMimesFromFile(join(this.repo.path, filePath))
+				return detectMimesFromFile(join(this.repo.path, filePath), false)
 					.then(result => result.mimes);
 			}
 
 			const child = this.repo.show(treeish + ':' + filePath);
 
 			return new TPromise<string[]>((c, e) =>
-				detectMimesFromStream(child.stdout, filePath)
+				detectMimesFromStream(child.stdout, filePath, false)
 					.then(result => result.mimes)
 			);
 		});

src/vs/workbench/services/files/electron-browser/fileService.ts

@@ -81,6 +81,7 @@ export class FileService implements IFileService {
 		const fileServiceConfig: IFileServiceOptions = {
 			errorLogger: (msg: string) => this.onFileServiceError(msg),
 			encoding: configuration.files && configuration.files.encoding,
+			autoDetectEncoding: configuration.files && configuration.files.autoDetectEncoding,
 			encodingOverride,
 			watcherIgnoredPatterns,
 			verboseLogging: environmentService.verbose,

src/vs/workbench/services/files/node/fileService.ts

@@ -44,6 +44,7 @@ export interface IFileServiceOptions {
 	tmpDir?: string;
 	errorLogger?: (msg: string) => void;
 	encoding?: string;
+	autoDetectEncoding?: boolean;
 	bom?: string;
 	encodingOverride?: IEncodingOverride[];
 	watcherIgnoredPatterns?: string[];
@@ -205,7 +206,7 @@ export class FileService implements IFileService {
 			}
 
 			// 2.) detect mimes
-			return mime.detectMimesFromFile(absolutePath).then((detected: mime.IMimeAndEncoding) => {
+			return mime.detectMimesFromFile(absolutePath, this.options.autoDetectEncoding).then((detected: mime.IMimeAndEncoding) => {
 				const isText = detected.mimes.indexOf(baseMime.MIME_BINARY) === -1;
 
 				// Return error early if client only accepts text and this is not text

src/vs/workbench/services/search/node/worker/searchWorker.ts

@@ -209,7 +209,7 @@ export class SearchWorkerEngine {
 
 						// Detect encoding and mime when this is the beginning of the file
 						if (isFirstRead) {
-							const mimeAndEncoding = detectMimeAndEncodingFromBuffer({ buffer, bytesRead });
+							const mimeAndEncoding = detectMimeAndEncodingFromBuffer({ buffer, bytesRead }, false);
 							if (mimeAndEncoding.mimes[mimeAndEncoding.mimes.length - 1] !== baseMime.MIME_TEXT) {
 								return clb(null); // skip files that seem binary
 							}