Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
掘金者说
vscode
提交
36708c0f
V
vscode
项目概览
掘金者说
/
vscode
与 Fork 源项目一致
从无法访问的项目Fork
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
V
vscode
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
36708c0f
编写于
4月 10, 2018
作者:
J
Johannes Rieken
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add `toDecodeStream` to enable seamless encoding handling, #41985
上级
82137280
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
194 addition
and
2 deletion
+194
-2
src/vs/base/node/encoding.ts
src/vs/base/node/encoding.ts
+81
-1
src/vs/base/test/node/encoding/encoding.test.ts
src/vs/base/test/node/encoding/encoding.test.ts
+113
-1
未找到文件。
src/vs/base/node/encoding.ts
浏览文件 @
36708c0f
...
...
@@ -10,12 +10,92 @@ import * as iconv from 'iconv-lite';
import
{
TPromise
}
from
'
vs/base/common/winjs.base
'
;
import
{
isLinux
,
isMacintosh
}
from
'
vs/base/common/platform
'
;
import
{
exec
}
from
'
child_process
'
;
import
{
Readable
,
Writable
,
WritableOptions
}
from
'
stream
'
;
export
const
UTF8
=
'
utf8
'
;
export
const
UTF8_with_bom
=
'
utf8bom
'
;
export
const
UTF16be
=
'
utf16be
'
;
export
const
UTF16le
=
'
utf16le
'
;
export
interface
IDecodeStreamOptions
{
minBytesRequiredForDetection
:
number
;
guessEncoding
:
boolean
;
overwriteEncoding
(
detected
:
string
):
string
;
}
export
function
toDecodeStream
(
readable
:
Readable
,
opts
:
IDecodeStreamOptions
):
TPromise
<
{
detected
:
IDetectedEncodingResult
,
stream
:
NodeJS
.
ReadableStream
}
>
{
return
new
TPromise
<
{
detected
:
IDetectedEncodingResult
,
stream
:
NodeJS
.
ReadableStream
}
>
((
resolve
,
reject
)
=>
{
readable
.
pipe
(
new
class
extends
Writable
{
private
_decodeStream
:
NodeJS
.
ReadWriteStream
;
private
_decodeStreamConstruction
:
Thenable
<
any
>
;
private
_buffer
:
Buffer
[]
=
[];
private
_bytesBuffered
=
0
;
constructor
(
opts
?:
WritableOptions
)
{
super
(
opts
);
this
.
once
(
'
finish
'
,
()
=>
this
.
_finish
());
}
_write
(
chunk
:
any
,
encoding
:
string
,
callback
:
Function
):
void
{
if
(
!
Buffer
.
isBuffer
(
chunk
))
{
callback
(
new
Error
(
'
data must be a buffer
'
));
}
if
(
this
.
_decodeStream
)
{
// just a forwarder now
this
.
_decodeStream
.
write
(
chunk
,
callback
);
return
;
}
this
.
_buffer
.
push
(
chunk
);
this
.
_bytesBuffered
+=
chunk
.
length
;
if
(
this
.
_decodeStreamConstruction
)
{
// waiting for the decoder to be ready
this
.
_decodeStreamConstruction
.
then
(
_
=>
callback
(),
err
=>
callback
(
err
));
}
else
if
(
this
.
_bytesBuffered
>=
opts
.
minBytesRequiredForDetection
)
{
// buffered enough data, create stream and forward data
this
.
_startDecodeStream
(
callback
);
}
else
{
// only buffering
callback
();
}
}
_startDecodeStream
(
callback
:
Function
):
void
{
this
.
_decodeStreamConstruction
=
TPromise
.
as
(
detectEncodingFromBuffer
({
buffer
:
Buffer
.
concat
(
this
.
_buffer
),
bytesRead
:
this
.
_bytesBuffered
},
opts
.
guessEncoding
)).
then
(
detected
=>
{
detected
.
encoding
=
opts
.
overwriteEncoding
(
detected
.
encoding
);
// default encoding
this
.
_decodeStream
=
decodeStream
(
detected
.
encoding
);
for
(
const
buffer
of
this
.
_buffer
)
{
this
.
_decodeStream
.
write
(
buffer
);
}
callback
();
resolve
({
detected
,
stream
:
this
.
_decodeStream
});
},
err
=>
{
callback
(
err
);
});
}
_finish
():
void
{
if
(
this
.
_decodeStream
)
{
// normal finish
this
.
_decodeStream
.
end
();
}
else
{
// we were still waiting for data...
this
.
_startDecodeStream
(()
=>
this
.
_decodeStream
.
end
());
}
}
});
});
}
export
function
bomLength
(
encoding
:
string
):
number
{
switch
(
encoding
)
{
case
UTF8
:
...
...
@@ -350,4 +430,4 @@ export function resolveTerminalEncoding(verbose?: boolean): TPromise<string> {
return
UTF8
;
});
}
\ No newline at end of file
}
src/vs/base/test/node/encoding/encoding.test.ts
浏览文件 @
36708c0f
...
...
@@ -6,9 +6,10 @@
'
use strict
'
;
import
*
as
assert
from
'
assert
'
;
import
*
as
fs
from
'
fs
'
;
import
*
as
encoding
from
'
vs/base/node/encoding
'
;
import
{
readExactlyByFile
}
from
'
vs/base/node/stream
'
;
import
{
Readable
}
from
'
stream
'
;
suite
(
'
Encoding
'
,
()
=>
{
test
(
'
detectBOM UTF-8
'
,
()
=>
{
...
...
@@ -150,4 +151,115 @@ suite('Encoding', () => {
});
});
});
async
function
readAndDecodeFromDisk
(
path
,
_encoding
)
{
return
new
Promise
<
string
>
((
resolve
,
reject
)
=>
{
fs
.
readFile
(
path
,
(
err
,
data
)
=>
{
if
(
err
)
{
reject
(
err
);
}
else
{
resolve
(
encoding
.
decode
(
data
,
_encoding
));
}
});
});
}
async
function
readAllAsString
(
stream
:
NodeJS
.
ReadableStream
)
{
return
new
Promise
<
string
>
((
resolve
,
reject
)
=>
{
let
all
=
''
;
stream
.
on
(
'
data
'
,
chunk
=>
{
all
+=
chunk
;
assert
.
equal
(
typeof
chunk
,
'
string
'
);
});
stream
.
on
(
'
end
'
,
()
=>
{
resolve
(
all
);
});
stream
.
on
(
'
error
'
,
reject
);
});
}
test
(
'
toDecodeStream - some stream
'
,
async
function
()
{
let
source
=
new
Readable
({
read
(
size
)
{
this
.
push
(
Buffer
.
from
([
65
,
66
,
67
]));
this
.
push
(
Buffer
.
from
([
65
,
66
,
67
]));
this
.
push
(
Buffer
.
from
([
65
,
66
,
67
]));
this
.
push
(
null
);
}
});
let
{
detected
,
stream
}
=
await
encoding
.
toDecodeStream
(
source
,
{
minBytesRequiredForDetection
:
4
,
guessEncoding
:
true
,
overwriteEncoding
()
{
return
encoding
.
UTF8
;
}
});
assert
.
ok
(
detected
);
assert
.
ok
(
stream
);
const
content
=
await
readAllAsString
(
stream
);
assert
.
equal
(
content
,
'
ABCABCABC
'
);
});
test
(
'
toDecodeStream - some stream, expect too much data
'
,
async
function
()
{
let
source
=
new
Readable
({
read
(
size
)
{
this
.
push
(
Buffer
.
from
([
65
,
66
,
67
]));
this
.
push
(
Buffer
.
from
([
65
,
66
,
67
]));
this
.
push
(
Buffer
.
from
([
65
,
66
,
67
]));
this
.
push
(
null
);
}
});
let
{
detected
,
stream
}
=
await
encoding
.
toDecodeStream
(
source
,
{
minBytesRequiredForDetection
:
64
,
guessEncoding
:
true
,
overwriteEncoding
()
{
return
encoding
.
UTF8
;
}
});
assert
.
ok
(
detected
);
assert
.
ok
(
stream
);
const
content
=
await
readAllAsString
(
stream
);
assert
.
equal
(
content
,
'
ABCABCABC
'
);
});
test
(
'
toDecodeStream - some stream, no data
'
,
async
function
()
{
let
source
=
new
Readable
({
read
(
size
)
{
this
.
push
(
null
);
// empty
}
});
let
{
detected
,
stream
}
=
await
encoding
.
toDecodeStream
(
source
,
{
minBytesRequiredForDetection
:
512
,
guessEncoding
:
true
,
overwriteEncoding
()
{
return
encoding
.
UTF8
;
}
});
assert
.
ok
(
detected
);
assert
.
ok
(
stream
);
const
content
=
await
readAllAsString
(
stream
);
assert
.
equal
(
content
,
''
);
});
test
(
'
toDecodeStream - encoding, utf16be
'
,
async
function
()
{
let
path
=
require
.
toUrl
(
'
./fixtures/some_utf16be.css
'
);
let
source
=
fs
.
createReadStream
(
path
);
let
{
detected
,
stream
}
=
await
encoding
.
toDecodeStream
(
source
,
{
minBytesRequiredForDetection
:
64
,
guessEncoding
:
true
,
overwriteEncoding
(
detected
)
{
return
detected
;
}
});
assert
.
equal
(
detected
.
encoding
,
'
utf16be
'
);
assert
.
equal
(
detected
.
seemsBinary
,
false
);
let
expected
=
await
readAndDecodeFromDisk
(
path
,
detected
.
encoding
);
let
actual
=
await
readAllAsString
(
stream
);
assert
.
equal
(
actual
,
expected
);
});
test
(
'
toDecodeStream - empty file
'
,
async
function
()
{
let
path
=
require
.
toUrl
(
'
./fixtures/empty.txt
'
);
let
source
=
fs
.
createReadStream
(
path
);
let
{
detected
,
stream
}
=
await
encoding
.
toDecodeStream
(
source
,
{
minBytesRequiredForDetection
:
64
,
guessEncoding
:
true
,
overwriteEncoding
()
{
return
encoding
.
UTF8
;
}
});
let
expected
=
await
readAndDecodeFromDisk
(
path
,
detected
.
encoding
);
let
actual
=
await
readAllAsString
(
stream
);
assert
.
equal
(
actual
,
expected
);
});
});
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录