Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Tonvia
TDengine
提交
34eb7e25
T
TDengine
项目概览
Tonvia
/
TDengine
与 Fork 源项目一致
Fork自
taosdata / TDengine
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
34eb7e25
编写于
4月 28, 2021
作者:
S
Steven Li
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Used Python generator to simplify crash_gen IPC processing of STDOUT/STDERR
上级
fbadcfb5
变更
1
隐藏空白更改
内联
并排
Showing
1 changed file
with
69 addition
and
64 deletion
+69
-64
tests/pytest/crash_gen/service_manager.py
tests/pytest/crash_gen/service_manager.py
+69
-64
未找到文件。
tests/pytest/crash_gen/service_manager.py
浏览文件 @
34eb7e25
...
@@ -9,7 +9,8 @@ import signal
...
@@ -9,7 +9,8 @@ import signal
import
logging
import
logging
import
time
import
time
from
subprocess
import
PIPE
,
Popen
,
TimeoutExpired
from
subprocess
import
PIPE
,
Popen
,
TimeoutExpired
from
typing
import
IO
,
List
,
NewType
,
Optional
from
typing
import
BinaryIO
,
Generator
,
IO
,
List
,
NewType
,
Optional
import
typing
try
:
try
:
import
psutil
import
psutil
...
@@ -275,11 +276,16 @@ class TdeSubProcess:
...
@@ -275,11 +276,16 @@ class TdeSubProcess:
return
'[TdeSubProc: pid = {}, status = {}]'
.
format
(
return
'[TdeSubProc: pid = {}, status = {}]'
.
format
(
self
.
getPid
(),
self
.
getStatus
()
)
self
.
getPid
(),
self
.
getStatus
()
)
def
getStdOut
(
self
):
def
getStdOut
(
self
)
->
BinaryIO
:
return
self
.
_popen
.
stdout
if
self
.
_popen
.
universal_newlines
:
# alias of text_mode
raise
CrashGenError
(
"We need binary mode for STDOUT IPC"
)
# Logging.info("Type of stdout is: {}".format(type(self._popen.stdout)))
return
typing
.
cast
(
BinaryIO
,
self
.
_popen
.
stdout
)
def
getStdErr
(
self
):
def
getStdErr
(
self
)
->
BinaryIO
:
return
self
.
_popen
.
stderr
if
self
.
_popen
.
universal_newlines
:
# alias of text_mode
raise
CrashGenError
(
"We need binary mode for STDERR IPC"
)
return
typing
.
cast
(
BinaryIO
,
self
.
_popen
.
stderr
)
# Now it's always running, since we matched the life cycle
# Now it's always running, since we matched the life cycle
# def isRunning(self):
# def isRunning(self):
...
@@ -846,7 +852,7 @@ class ServiceManagerThread:
...
@@ -846,7 +852,7 @@ class ServiceManagerThread:
def
procIpcBatch
(
self
,
trimToTarget
=
0
,
forceOutput
=
False
):
def
procIpcBatch
(
self
,
trimToTarget
=
0
,
forceOutput
=
False
):
'''
'''
Process a batch of STDOUT/STDERR data, until we read EMPTY from
Process a batch of STDOUT/STDERR data, until we read EMPTY from
the
pip
e.
the
queu
e.
'''
'''
self
.
_trimQueue
(
trimToTarget
)
# trim if necessary
self
.
_trimQueue
(
trimToTarget
)
# trim if necessary
# Process all the output generated by the underlying sub process,
# Process all the output generated by the underlying sub process,
...
@@ -876,79 +882,78 @@ class ServiceManagerThread:
...
@@ -876,79 +882,78 @@ class ServiceManagerThread:
print
(
pBar
,
end
=
""
,
flush
=
True
)
print
(
pBar
,
end
=
""
,
flush
=
True
)
print
(
'
\b\b\b\b
'
,
end
=
""
,
flush
=
True
)
print
(
'
\b\b\b\b
'
,
end
=
""
,
flush
=
True
)
BinaryLine
=
NewType
(
'BinaryLine'
,
bytes
)
# line with binary data, directly from STDOUT, etc.
BinaryChunk
=
NewType
(
'BinaryChunk'
,
bytes
)
# line with binary data, directly from STDOUT, etc.
TextLine
=
NewType
(
'TextLine'
,
str
)
# properly decoded, suitable for printing, etc.
TextChunk
=
NewType
(
'TextChunk'
,
str
)
# properly decoded, suitable for printing, etc.
x
=
TextLine
(
'xyz'
)
@
classmethod
@
classmethod
def
_decodeBin
Line
(
cls
,
bLine
:
BinaryLine
)
->
Optional
[
TextLine
]
:
def
_decodeBin
aryChunk
(
cls
,
bChunk
:
bytes
)
->
Optional
[
TextChunk
]
:
try
:
try
:
t
Line
=
bLine
.
decode
(
"utf-8"
).
rstrip
()
t
Chunk
=
bChunk
.
decode
(
"utf-8"
).
rstrip
()
return
cls
.
Text
Line
(
tLine
)
return
cls
.
Text
Chunk
(
tChunk
)
except
UnicodeError
:
except
UnicodeError
:
print
(
"
\n
Non-UTF8 server output: {}
\n
"
.
format
(
b
Line
.
decode
(
'cp437'
)))
print
(
"
\n
Non-UTF8 server output: {}
\n
"
.
format
(
b
Chunk
.
decode
(
'cp437'
)))
return
None
return
None
def
svcOutputReader
(
self
,
out
:
IO
,
queue
,
logDir
:
str
):
def
_textChunkGenerator
(
self
,
streamIn
:
BinaryIO
,
logDir
:
str
,
logFile
:
str
)
->
Generator
[
TextChunk
,
None
,
None
]:
'''
Take an input stream with binary data, produced a generator of decoded
"text chunks", and also save the original binary data in a log file.
'''
os
.
makedirs
(
logDir
,
exist_ok
=
True
)
logF
=
open
(
os
.
path
.
join
(
logDir
,
logFile
),
'wb'
)
for
bChunk
in
iter
(
streamIn
.
readline
,
b
''
):
logF
.
write
(
bChunk
)
# Write to log file immediately
tChunk
=
self
.
_decodeBinaryChunk
(
bChunk
)
# decode
if
tChunk
is
not
None
:
yield
tChunk
# TODO: split into actual text lines
# At the end...
streamIn
.
close
()
# Close the stream
logF
.
close
()
# Close the output file
def
svcOutputReader
(
self
,
stdOut
:
BinaryIO
,
queue
,
logDir
:
str
):
'''
'''
The infinite routine that processes the STDOUT stream for the sub process being managed.
The infinite routine that processes the STDOUT stream for the sub process being managed.
:param
o
ut: the IO stream object used to fetch the data from
:param
stdO
ut: the IO stream object used to fetch the data from
:param queue: the queue where we dump the roughly parsed
line-by-line
data
:param queue: the queue where we dump the roughly parsed
chunk-by-chunk text
data
:param logDir: where we should dump a verbatim output file
:param logDir: where we should dump a verbatim output file
'''
'''
os
.
makedirs
(
logDir
,
exist_ok
=
True
)
logFile
=
os
.
path
.
join
(
logDir
,
'stdout.log'
)
fOut
=
open
(
logFile
,
'wb'
)
# Important Reference: https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python
# Important Reference: https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python
# print("This is the svcOutput Reader...")
# print("This is the svcOutput Reader...")
# for line in out :
# stdOut.readline() # Skip the first output? TODO: remove?
out
.
readline
()
for
tChunk
in
self
.
_textChunkGenerator
(
stdOut
,
logDir
,
'stdout.log'
)
:
for
bLine
in
iter
(
out
.
readline
,
b
''
):
queue
.
put
(
tChunk
)
# tChunk garanteed not to be None
fOut
.
write
(
bLine
)
self
.
_printProgress
(
"_i"
)
# print("Finished reading a line: {}".format(line))
# print("Adding item to queue...")
if
self
.
_status
.
isStarting
():
# we are starting, let's see if we have started
if
tChunk
.
find
(
self
.
TD_READY_MSG
)
!=
-
1
:
# found
# Moved to above
Logging
.
info
(
"Waiting for the service to become FULLY READY"
)
# try:
time
.
sleep
(
1.0
)
# wait for the server to truly start. TODO: remove this
# line = line.decode("utf-8").rstrip()
Logging
.
info
(
"Service is now FULLY READY"
)
# TODO: more ID info here?
# except UnicodeError:
self
.
_status
.
set
(
Status
.
STATUS_RUNNING
)
# print("\nNon-UTF8 server output: {}\n".format(line))
tLine
=
self
.
_decodeBinLine
(
bLine
)
# Trim the queue if necessary: TODO: try this 1 out of 10 times
self
.
_trimQueue
(
self
.
MAX_QUEUE_SIZE
*
9
//
10
)
# trim to 90% size
if
tLine
is
not
None
:
# This might block, and then causing "out" buffer to block
if
self
.
_status
.
isStopping
():
# TODO: use thread status instead
queue
.
put
(
tLine
)
# WAITING for stopping sub process to finish its outptu
self
.
_printProgress
(
"_i"
)
print
(
"_w"
,
end
=
""
,
flush
=
True
)
if
self
.
_status
.
isStarting
():
# we are starting, let's see if we have started
if
tLine
.
find
(
self
.
TD_READY_MSG
)
!=
-
1
:
# found
Logging
.
info
(
"Waiting for the service to become FULLY READY"
)
time
.
sleep
(
1.0
)
# wait for the server to truly start. TODO: remove this
Logging
.
info
(
"Service is now FULLY READY"
)
# TODO: more ID info here?
self
.
_status
.
set
(
Status
.
STATUS_RUNNING
)
# Trim the queue if necessary: TODO: try this 1 out of 10 times
self
.
_trimQueue
(
self
.
MAX_QUEUE_SIZE
*
9
//
10
)
# trim to 90% size
if
self
.
_status
.
isStopping
():
# TODO: use thread status instead
# WAITING for stopping sub process to finish its outptu
print
(
"_w"
,
end
=
""
,
flush
=
True
)
# queue.put(line)
# queue.put(line)
# meaning sub process must have died
#
stdOut has no more data,
meaning sub process must have died
Logging
.
info
(
"EOF found TDengine STDOUT, marking the process as terminated"
)
Logging
.
info
(
"EOF found TDengine STDOUT, marking the process as terminated"
)
self
.
setStatus
(
Status
.
STATUS_STOPPED
)
self
.
setStatus
(
Status
.
STATUS_STOPPED
)
out
.
close
()
# Close the stream
fOut
.
close
()
# Close the output file
def
svcErrorReader
(
self
,
err
:
IO
,
queue
,
logDir
:
str
):
def
svcErrorReader
(
self
,
stdErr
:
BinaryIO
,
queue
,
logDir
:
str
):
os
.
makedirs
(
logDir
,
exist_ok
=
True
)
# os.makedirs(logDir, exist_ok=True)
logFile
=
os
.
path
.
join
(
logDir
,
'stderr.log'
)
# logFile = os.path.join(logDir,'stderr.log')
fErr
=
open
(
logFile
,
'wb'
)
# fErr = open(logFile, 'wb')
for
line
in
iter
(
err
.
readline
,
b
''
):
# for line in iter(err.readline, b''):
fErr
.
write
(
line
)
for
tChunk
in
self
.
_textChunkGenerator
(
stdErr
,
logDir
,
'stderr.log'
)
:
Logging
.
info
(
"TDengine STDERR: {}"
.
format
(
line
))
queue
.
put
(
tChunk
)
# tChunk garanteed not to be None
# fErr.write(line)
Logging
.
info
(
"TDengine STDERR: {}"
.
format
(
tChunk
))
Logging
.
info
(
"EOF for TDengine STDERR"
)
Logging
.
info
(
"EOF for TDengine STDERR"
)
err
.
close
()
fErr
.
close
()
\ No newline at end of file
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录