Skip to content

V
vscode

xxadev / vscode
与 Fork 源项目一致

从无法访问的项目Fork

通知 2
Star 0
Fork 0
V
vscode

体验新版 GitCode，发现更多精彩内容 >>

提交 498325ce 编写于 作者: B Benjamin Pasero
浏览文件
操作

Load iconv-lite module async (fix #40147)

上级 3edd0f95
隐藏空白更改
内联 并排
Showing with 59 addition and 58 deletion
src/vs/base/node/encoding.ts
浏览文件 @ 498325ce
......@@ -3,7 +3,7 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as iconv from 'iconv-lite';
import { DecoderStream } from 'iconv-lite';
import { Readable, ReadableStream, newWriteableStream } from 'vs/base/common/stream';
import { VSBuffer, VSBufferReadable, VSBufferReadableStream } from 'vs/base/common/buffer';
......@@ -31,7 +31,7 @@ export interface IDecodeStreamOptions {
guessEncoding: boolean;
minBytesRequiredForDetection?: number;
overwriteEncoding(detectedEncoding: string | null): string;
overwriteEncoding(detectedEncoding: string | null): Promise<string>;
}
export interface IDecodeStreamResult {
......@@ -48,7 +48,7 @@ export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeS
const bufferedChunks: VSBuffer[] = [];
let bytesBuffered = 0;
let decoder: iconv.DecoderStream | undefined = undefined;
let decoder: DecoderStream | undefined = undefined;
const createDecoder = async () => {
try {
......@@ -60,9 +60,10 @@ export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeS
}, options.guessEncoding);
// ensure to respect overwrite of encoding
detected.encoding = options.overwriteEncoding(detected.encoding);
detected.encoding = await options.overwriteEncoding(detected.encoding);
// decode and write buffered content
const iconv = await import('iconv-lite');
decoder = iconv.getDecoder(toNodeEncoding(detected.encoding));
const decoded = decoder.write(Buffer.from(VSBuffer.concat(bufferedChunks).buffer));
target.write(decoded);
......@@ -127,8 +128,10 @@ export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeS
});
}
export function toEncodeReadable(readable: Readable<string>, encoding: string, options?: { addBOM?: boolean }): VSBufferReadable {
export async function toEncodeReadable(readable: Readable<string>, encoding: string, options?: { addBOM?: boolean }): Promise<VSBufferReadable> {
const iconv = await import('iconv-lite');
const encoder = iconv.getEncoder(toNodeEncoding(encoding), options);
let bytesRead = 0;
let done = false;
......@@ -172,7 +175,9 @@ export function toEncodeReadable(readable: Readable<string>, encoding: string, o
};
}
export function encodingExists(encoding: string): boolean {
export async function encodingExists(encoding: string): Promise<boolean> {
const iconv = await import('iconv-lite');
return iconv.encodingExists(toNodeEncoding(encoding));
}
......
src/vs/base/test/node/encoding/encoding.test.ts
浏览文件 @ 498325ce
......@@ -129,7 +129,7 @@ suite('Encoding', () => {
process.env['VSCODE_CLI_ENCODING'] = 'utf16le';
const enc = await terminalEncoding.resolveTerminalEncoding();
assert.ok(encoding.encodingExists(enc));
assert.ok(await encoding.encodingExists(enc));
assert.equal(enc, 'utf16le');
});
......@@ -252,14 +252,13 @@ suite('Encoding', () => {
}
test('toDecodeStream - some stream', async function () {
let source = newTestReadableStream([
const source = newTestReadableStream([
Buffer.from([65, 66, 67]),
Buffer.from([65, 66, 67]),
Buffer.from([65, 66, 67]),
]);
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
const { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 });
assert.ok(detected);
assert.ok(stream);
......@@ -269,14 +268,13 @@ suite('Encoding', () => {
});
test('toDecodeStream - some stream, expect too much data', async function () {
let source = newTestReadableStream([
const source = newTestReadableStream([
Buffer.from([65, 66, 67]),
Buffer.from([65, 66, 67]),
Buffer.from([65, 66, 67]),
]);
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
const { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 });
assert.ok(detected);
assert.ok(stream);
......@@ -286,11 +284,10 @@ suite('Encoding', () => {
});
test('toDecodeStream - some stream, no data', async function () {
let source = newWriteableBufferStream();
const source = newWriteableBufferStream();
source.end();
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 512, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
const { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 512, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 });
assert.ok(detected);
assert.ok(stream);
......@@ -301,60 +298,58 @@ suite('Encoding', () => {
test('toDecodeStream - encoding, utf16be', async function () {
const path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
const source = streamToBufferReadableStream(fs.createReadStream(path));
let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
let source = streamToBufferReadableStream(fs.createReadStream(path));
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
const { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 });
assert.equal(detected.encoding, 'utf16be');
assert.equal(detected.seemsBinary, false);
let expected = await readAndDecodeFromDisk(path, detected.encoding);
let actual = await readAllAsString(stream);
const expected = await readAndDecodeFromDisk(path, detected.encoding);
const actual = await readAllAsString(stream);
assert.equal(actual, expected);
});
test('toDecodeStream - empty file', async function () {
const path = getPathFromAmdModule(require, './fixtures/empty.txt');
const source = streamToBufferReadableStream(fs.createReadStream(path));
const { detected, stream } = await encoding.toDecodeStream(source, { guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 });
let path = getPathFromAmdModule(require, './fixtures/empty.txt');
let source = streamToBufferReadableStream(fs.createReadStream(path));
let { detected, stream } = await encoding.toDecodeStream(source, { guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
let expected = await readAndDecodeFromDisk(path, detected.encoding);
let actual = await readAllAsString(stream);
const expected = await readAndDecodeFromDisk(path, detected.encoding);
const actual = await readAllAsString(stream);
assert.equal(actual, expected);
});
test('toDecodeStream - decodes buffer entirely', async function () {
let emojis = Buffer.from('🖥️💻💾');
let incompleteEmojis = emojis.slice(0, emojis.length - 1);
const emojis = Buffer.from('🖥️💻💾');
const incompleteEmojis = emojis.slice(0, emojis.length - 1);
let buffers = [];
const buffers: Buffer[] = [];
for (let i = 0; i < incompleteEmojis.length; i++) {
buffers.push(incompleteEmojis.slice(i, i + 1));
}
const source = newTestReadableStream(buffers);
let { stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
const { stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 });
let expected = incompleteEmojis.toString(encoding.UTF8);
let actual = await readAllAsString(stream);
const expected = incompleteEmojis.toString(encoding.UTF8);
const actual = await readAllAsString(stream);
assert.equal(actual, expected);
});
test('toEncodeReadable - encoding, utf16be', async function () {
const path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
const source = await readAndDecodeFromDisk(path, encoding.UTF16be);
let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
let source = await readAndDecodeFromDisk(path, encoding.UTF16be);
let expected = VSBuffer.wrap(
const expected = VSBuffer.wrap(
iconv.encode(source, encoding.toNodeEncoding(encoding.UTF16be))
).toString();
let actual = streams.consumeReadable(
encoding.toEncodeReadable(streams.toReadable(source), encoding.UTF16be),
const actual = streams.consumeReadable(
await encoding.toEncodeReadable(streams.toReadable(source), encoding.UTF16be),
VSBuffer.concat
).toString();
......@@ -362,15 +357,14 @@ suite('Encoding', () => {
});
test('toEncodeReadable - empty readable to utf8', async function () {
const source: streams.Readable<string> = {
read() {
return null;
}
};
let actual = streams.consumeReadable(
encoding.toEncodeReadable(source, encoding.UTF8),
const actual = streams.consumeReadable(
await encoding.toEncodeReadable(source, encoding.UTF8),
VSBuffer.concat
).toString();
......@@ -391,17 +385,16 @@ suite('Encoding', () => {
relatedBom: encoding.UTF16le_BOM
}].forEach(({ utfEncoding, relatedBom }) => {
test(`toEncodeReadable - empty readable to ${utfEncoding} with BOM`, async function () {
const source: streams.Readable<string> = {
read() {
return null;
}
};
let encodedReadable = encoding.toEncodeReadable(source, utfEncoding, { addBOM: true });
const encodedReadable = encoding.toEncodeReadable(source, utfEncoding, { addBOM: true });
const expected = VSBuffer.wrap(Buffer.from(relatedBom)).toString();
const actual = streams.consumeReadable(encodedReadable, VSBuffer.concat).toString();
const actual = streams.consumeReadable(await encodedReadable, VSBuffer.concat).toString();
assert.equal(actual, expected);
});
......
src/vs/workbench/services/textfile/browser/browserTextFileService.ts
浏览文件 @ 498325ce
......@@ -11,7 +11,7 @@ import { ShutdownReason } from 'vs/platform/lifecycle/common/lifecycle';
export class BrowserTextFileService extends AbstractTextFileService {
readonly encoding: IResourceEncodings = {
getPreferredWriteEncoding(): IResourceEncoding {
async getPreferredWriteEncoding(): Promise<IResourceEncoding> {
return { encoding: 'utf8', hasBOM: false };
}
};
......
src/vs/workbench/services/textfile/common/textFileEditorModel.ts
浏览文件 @ 498325ce
......@@ -286,6 +286,7 @@ export class TextFileEditorModel extends BaseTextEditorModel implements ITextFil
}
private async loadFromBackup(backup: IResolvedBackup<IBackupMetaData>, options?: ITextFileLoadOptions): Promise<TextFileEditorModel> {
const preferredEncoding = await this.textFileService.encoding.getPreferredWriteEncoding(this.resource, this.preferredEncoding);
// Load with backup
this.loadFromContent({
......@@ -296,7 +297,7 @@ export class TextFileEditorModel extends BaseTextEditorModel implements ITextFil
size: backup.meta ? backup.meta.size : 0,
etag: backup.meta ? backup.meta.etag : ETAG_DISABLED, // etag disabled if unknown!
value: backup.value,
encoding: this.textFileService.encoding.getPreferredWriteEncoding(this.resource, this.preferredEncoding).encoding
encoding: preferredEncoding.encoding
}, options, true /* from backup */);
// Restore orphaned flag based on state
......
src/vs/workbench/services/textfile/common/textfiles.ts
浏览文件 @ 498325ce
......@@ -171,7 +171,7 @@ export class TextFileOperationError extends FileOperationError {
}
export interface IResourceEncodings {
getPreferredWriteEncoding(resource: URI, preferredEncoding?: string): IResourceEncoding;
getPreferredWriteEncoding(resource: URI, preferredEncoding?: string): Promise<IResourceEncoding>;
}
export interface IResourceEncoding {
......
src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts
浏览文件 @ 498325ce
......@@ -170,7 +170,8 @@ export class NativeTextFileService extends AbstractTextFileService {
}
// otherwise create with encoding
return this.fileService.createFile(resource, this.getEncodedReadable(value || '', encoding, addBOM), options);
const encodedReadable = await this.getEncodedReadable(value || '', encoding, addBOM);
return this.fileService.createFile(resource, encodedReadable, options);
}
async write(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<IFileStatWithMetadata> {
......@@ -204,7 +205,8 @@ export class NativeTextFileService extends AbstractTextFileService {
// otherwise save with encoding
else {
return await this.fileService.writeFile(resource, this.getEncodedReadable(value, encoding, addBOM), options);
const encodedReadable = await this.getEncodedReadable(value, encoding, addBOM);
return await this.fileService.writeFile(resource, encodedReadable, options);
}
} catch (error) {
......@@ -229,7 +231,7 @@ export class NativeTextFileService extends AbstractTextFileService {
}
}
private getEncodedReadable(value: string | ITextSnapshot, encoding: string, addBOM: boolean): VSBufferReadable {
private getEncodedReadable(value: string | ITextSnapshot, encoding: string, addBOM: boolean): Promise<VSBufferReadable> {
const snapshot = typeof value === 'string' ? stringToSnapshot(value) : value;
return toEncodeReadable(snapshot, encoding, { addBOM });
}
......@@ -340,7 +342,7 @@ export class EncodingOracle extends Disposable implements IResourceEncodings {
}
async getWriteEncoding(resource: URI, options?: IWriteTextFileOptions): Promise<{ encoding: string, addBOM: boolean }> {
const { encoding, hasBOM } = this.getPreferredWriteEncoding(resource, options ? options.encoding : undefined);
const { encoding, hasBOM } = await this.getPreferredWriteEncoding(resource, options ? options.encoding : undefined);
// Some encodings come with a BOM automatically
if (hasBOM) {
......@@ -364,8 +366,8 @@ export class EncodingOracle extends Disposable implements IResourceEncodings {
return { encoding, addBOM: false };
}
getPreferredWriteEncoding(resource: URI, preferredEncoding?: string): IResourceEncoding {
const resourceEncoding = this.getEncodingForResource(resource, preferredEncoding);
async getPreferredWriteEncoding(resource: URI, preferredEncoding?: string): Promise<IResourceEncoding> {
const resourceEncoding = await this.getEncodingForResource(resource, preferredEncoding);
return {
encoding: resourceEncoding,
......@@ -373,7 +375,7 @@ export class EncodingOracle extends Disposable implements IResourceEncodings {
};
}
getReadEncoding(resource: URI, options: IReadTextFileOptions | undefined, detectedEncoding: string | null): string {
getReadEncoding(resource: URI, options: IReadTextFileOptions | undefined, detectedEncoding: string | null): Promise<string> {
let preferredEncoding: string | undefined;
// Encoding passed in as option
......@@ -398,7 +400,7 @@ export class EncodingOracle extends Disposable implements IResourceEncodings {
return this.getEncodingForResource(resource, preferredEncoding);
}
private getEncodingForResource(resource: URI, preferredEncoding?: string): string {
private async getEncodingForResource(resource: URI, preferredEncoding?: string): Promise<string> {
let fileEncoding: string;
const override = this.getEncodingOverride(resource);
......@@ -410,7 +412,7 @@ export class EncodingOracle extends Disposable implements IResourceEncodings {
fileEncoding = this.textResourceConfigurationService.getValue(resource, 'files.encoding'); // and last we check for settings
}
if (!fileEncoding || !encodingExists(fileEncoding)) {
if (!fileEncoding || !(await encodingExists(fileEncoding))) {
fileEncoding = UTF8; // the default is UTF 8
}
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或