Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
6600540f
TDengine
项目概览
taosdata
/
TDengine
12 个月 前同步成功
通知
1181
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
6600540f
编写于
7月 26, 2022
作者:
H
Haojun Liao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refactor: optimize the memory consumption during tsdbread
上级
231f4399
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
12 addition
and
8 deletion
+12
-8
source/dnode/vnode/src/inc/tsdb.h
source/dnode/vnode/src/inc/tsdb.h
+1
-1
source/dnode/vnode/src/tsdb/tsdbRead.c
source/dnode/vnode/src/tsdb/tsdbRead.c
+11
-7
未找到文件。
source/dnode/vnode/src/inc/tsdb.h
浏览文件 @
6600540f
...
...
@@ -371,8 +371,8 @@ struct SBlockIdx {
struct
SMapData
{
int32_t
nItem
;
int32_t
*
aOffset
;
int32_t
nData
;
int32_t
*
aOffset
;
uint8_t
*
pData
;
};
...
...
source/dnode/vnode/src/tsdb/tsdbRead.c
浏览文件 @
6600540f
...
...
@@ -31,7 +31,6 @@ typedef struct {
typedef
struct
STableBlockScanInfo
{
uint64_t
uid
;
TSKEY
lastKey
;
SBlockIdx
blockIdx
;
SMapData
mapData
;
// block info (compressed)
SArray
*
pBlockList
;
// block data index list
SIterInfo
iter
;
// mem buffer skip list iterator
...
...
@@ -188,7 +187,7 @@ static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) {
static
SHashObj
*
createDataBlockScanInfo
(
STsdbReader
*
pTsdbReader
,
const
STableKeyInfo
*
idList
,
int32_t
numOfTables
)
{
// allocate buffer in order to load data blocks from file
// todo use simple hash instead
// todo use simple hash instead
, optimize the memory consumption
SHashObj
*
pTableMap
=
taosHashInit
(
numOfTables
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BIGINT
),
false
,
HASH_NO_LOCK
);
if
(
pTableMap
==
NULL
)
{
...
...
@@ -249,6 +248,7 @@ static void destroyBlockScanInfo(SHashObj* pTableMap) {
p
->
delSkyline
=
taosArrayDestroy
(
p
->
delSkyline
);
p
->
pBlockList
=
taosArrayDestroy
(
p
->
pBlockList
);
tMapDataClear
(
&
p
->
mapData
);
}
taosHashCleanup
(
pTableMap
);
...
...
@@ -565,7 +565,6 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader,
pScanInfo
->
pBlockList
=
taosArrayInit
(
4
,
sizeof
(
int32_t
));
}
pScanInfo
->
blockIdx
=
*
pBlockIdx
;
taosArrayPush
(
pIndexList
,
pBlockIdx
);
}
...
...
@@ -775,7 +774,8 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
SBlockLoadSuppInfo
*
pSupInfo
=
&
pReader
->
suppInfo
;
SFileBlockDumpInfo
*
pDumpInfo
=
&
pReader
->
status
.
fBlockDumpInfo
;
int32_t
code
=
tsdbReadColData
(
pReader
->
pFileReader
,
&
pBlockScanInfo
->
blockIdx
,
pBlock
,
pSupInfo
->
colIds
,
numOfCols
,
SBlockIdx
blockIdx
=
{.
suid
=
pReader
->
suid
,
.
uid
=
pBlockScanInfo
->
uid
};
int32_t
code
=
tsdbReadColData
(
pReader
->
pFileReader
,
&
blockIdx
,
pBlock
,
pSupInfo
->
colIds
,
numOfCols
,
pBlockData
,
NULL
,
NULL
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
goto
_error
;
...
...
@@ -904,10 +904,10 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte
}
sup
.
pDataBlockInfo
[
sup
.
numOfTables
]
=
(
SBlockOrderWrapper
*
)
buf
;
SBlock
block
=
{
0
};
for
(
int32_t
k
=
0
;
k
<
num
;
++
k
)
{
SBlockOrderWrapper
wrapper
=
{
0
};
SBlock
block
=
{
0
};
int32_t
*
mapDataIndex
=
taosArrayGet
(
pTableScanInfo
->
pBlockList
,
k
);
tMapDataGetItemByIdx
(
&
pTableScanInfo
->
mapData
,
*
mapDataIndex
,
&
block
,
tGetBlock
);
...
...
@@ -2563,6 +2563,8 @@ void tsdbReaderClose(STsdbReader* pReader) {
taosMemoryFree
(
pSupInfo
->
buildBuf
);
cleanupDataBlockIterator
(
&
pReader
->
status
.
blockIter
);
size_t
numOfTables
=
taosHashGetSize
(
pReader
->
status
.
pTableMap
);
destroyBlockScanInfo
(
pReader
->
status
.
pTableMap
);
blockDataDestroy
(
pReader
->
pResBlock
);
...
...
@@ -2573,9 +2575,11 @@ void tsdbReaderClose(STsdbReader* pReader) {
SIOCostSummary
*
pCost
=
&
pReader
->
cost
;
tsdbDebug
(
"%p :io-cost summary: head-file:%"
PRIu64
", head-file time:%.2f ms, SMA:%"
PRId64
" SMA-time:%.2f ms, "
"fileBlocks:%"
PRId64
", fileBlocks-time:%.2f ms, build in-memory-block-time:%.2f ms, %s"
,
"fileBlocks:%"
PRId64
", fileBlocks-time:%.2f ms, build in-memory-block-time:%.2f ms, STableBlockScanInfo "
"size:%.2f Kb %s"
,
pReader
,
pCost
->
headFileLoad
,
pCost
->
headFileLoadTime
,
pCost
->
smaData
,
pCost
->
smaLoadTime
,
pCost
->
numOfBlocks
,
pCost
->
blockLoadTime
,
pCost
->
buildmemBlock
,
pReader
->
idStr
);
pCost
->
numOfBlocks
,
pCost
->
blockLoadTime
,
pCost
->
buildmemBlock
,
numOfTables
*
sizeof
(
STableBlockScanInfo
)
/
1000
.
0
,
pReader
->
idStr
);
taosMemoryFree
(
pReader
->
idStr
);
taosMemoryFree
(
pReader
->
pSchema
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录