Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
89f7ad29
T
TDengine
项目概览
taosdata
/
TDengine
12 个月 前同步成功
通知
1181
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
89f7ad29
编写于
10月 25, 2022
作者:
M
Minglei Jin
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix: optimize tsdb cache loading speed
上级
a01d49b2
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
188 addition
and
147 deletion
+188
-147
source/dnode/vnode/inc/vnode.h
source/dnode/vnode/inc/vnode.h
+2
-1
source/dnode/vnode/src/inc/tsdb.h
source/dnode/vnode/src/inc/tsdb.h
+38
-23
source/dnode/vnode/src/tsdb/tsdbCache.c
source/dnode/vnode/src/tsdb/tsdbCache.c
+114
-97
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
+26
-20
source/libs/executor/src/cachescanoperator.c
source/libs/executor/src/cachescanoperator.c
+8
-6
未找到文件。
source/dnode/vnode/inc/vnode.h
浏览文件 @
89f7ad29
...
...
@@ -163,7 +163,8 @@ void *tsdbGetIdx(SMeta *pMeta);
void
*
tsdbGetIvtIdx
(
SMeta
*
pMeta
);
uint64_t
getReaderMaxVersion
(
STsdbReader
*
pReader
);
int32_t
tsdbCacherowsReaderOpen
(
void
*
pVnode
,
int32_t
type
,
SArray
*
pTableIdList
,
int32_t
numOfCols
,
void
**
pReader
);
int32_t
tsdbCacherowsReaderOpen
(
void
*
pVnode
,
int32_t
type
,
SArray
*
pTableIdList
,
int32_t
numOfCols
,
uint64_t
suid
,
void
**
pReader
);
int32_t
tsdbRetrieveCacheRows
(
void
*
pReader
,
SSDataBlock
*
pResBlock
,
const
int32_t
*
slotIds
,
SArray
*
pTableUids
);
void
*
tsdbCacherowsReaderClose
(
void
*
pReader
);
int32_t
tsdbGetTableSchema
(
SVnode
*
pVnode
,
int64_t
uid
,
STSchema
**
pSchema
,
int64_t
*
suid
);
...
...
source/dnode/vnode/src/inc/tsdb.h
浏览文件 @
89f7ad29
...
...
@@ -298,29 +298,6 @@ int32_t tsdbMerge(STsdb *pTsdb);
#define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0)
#define TSDB_CACHE_LAST(c) (((c).cacheLast & 2) > 0)
// tsdbCache ==============================================================================================
typedef
struct
{
TSKEY
ts
;
SColVal
colVal
;
}
SLastCol
;
int32_t
tsdbOpenCache
(
STsdb
*
pTsdb
);
void
tsdbCloseCache
(
STsdb
*
pTsdb
);
int32_t
tsdbCacheInsertLast
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
STSRow
*
row
,
STsdb
*
pTsdb
);
int32_t
tsdbCacheInsertLastrow
(
SLRUCache
*
pCache
,
STsdb
*
pTsdb
,
tb_uid_t
uid
,
STSRow
*
row
,
bool
dup
);
int32_t
tsdbCacheGetLastH
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
STsdb
*
pTsdb
,
LRUHandle
**
h
);
int32_t
tsdbCacheGetLastrowH
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
STsdb
*
pTsdb
,
LRUHandle
**
h
);
int32_t
tsdbCacheRelease
(
SLRUCache
*
pCache
,
LRUHandle
*
h
);
int32_t
tsdbCacheDeleteLastrow
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
TSKEY
eKey
);
int32_t
tsdbCacheDeleteLast
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
TSKEY
eKey
);
int32_t
tsdbCacheDelete
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
TSKEY
eKey
);
void
tsdbCacheSetCapacity
(
SVnode
*
pVnode
,
size_t
capacity
);
size_t
tsdbCacheGetCapacity
(
SVnode
*
pVnode
);
int32_t
tsdbCacheLastArray2Row
(
SArray
*
pLastArray
,
STSRow
**
ppRow
,
STSchema
*
pSchema
);
// tsdbDiskData ==============================================================================================
int32_t
tDiskDataBuilderCreate
(
SDiskDataBuilder
**
ppBuilder
);
void
*
tDiskDataBuilderDestroy
(
SDiskDataBuilder
*
pBuilder
);
...
...
@@ -729,6 +706,44 @@ void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
void
getLastBlockLoadInfo
(
SSttBlockLoadInfo
*
pLoadInfo
,
int64_t
*
blocks
,
double
*
el
);
void
*
destroyLastBlockLoadInfo
(
SSttBlockLoadInfo
*
pLoadInfo
);
// tsdbCache ==============================================================================================
typedef
struct
SCacheRowsReader
{
SVnode
*
pVnode
;
STSchema
*
pSchema
;
uint64_t
uid
;
uint64_t
suid
;
char
**
transferBuf
;
// todo remove it soon
int32_t
numOfCols
;
int32_t
type
;
int32_t
tableIndex
;
// currently returned result tables
SArray
*
pTableList
;
// table id list
SSttBlockLoadInfo
*
pLoadInfo
;
STsdbReadSnap
*
pReadSnap
;
SDataFReader
*
pDataFReader
;
}
SCacheRowsReader
;
typedef
struct
{
TSKEY
ts
;
SColVal
colVal
;
}
SLastCol
;
int32_t
tsdbOpenCache
(
STsdb
*
pTsdb
);
void
tsdbCloseCache
(
STsdb
*
pTsdb
);
int32_t
tsdbCacheInsertLast
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
STSRow
*
row
,
STsdb
*
pTsdb
);
int32_t
tsdbCacheInsertLastrow
(
SLRUCache
*
pCache
,
STsdb
*
pTsdb
,
tb_uid_t
uid
,
STSRow
*
row
,
bool
dup
);
int32_t
tsdbCacheGetLastH
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
SCacheRowsReader
*
pr
,
LRUHandle
**
h
);
int32_t
tsdbCacheGetLastrowH
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
SCacheRowsReader
*
pr
,
LRUHandle
**
h
);
int32_t
tsdbCacheRelease
(
SLRUCache
*
pCache
,
LRUHandle
*
h
);
int32_t
tsdbCacheDeleteLastrow
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
TSKEY
eKey
);
int32_t
tsdbCacheDeleteLast
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
TSKEY
eKey
);
int32_t
tsdbCacheDelete
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
TSKEY
eKey
);
void
tsdbCacheSetCapacity
(
SVnode
*
pVnode
,
size_t
capacity
);
size_t
tsdbCacheGetCapacity
(
SVnode
*
pVnode
);
int32_t
tsdbCacheLastArray2Row
(
SArray
*
pLastArray
,
STSRow
**
ppRow
,
STSchema
*
pSchema
);
// ========== inline functions ==========
static
FORCE_INLINE
int32_t
tsdbKeyCmprFn
(
const
void
*
p1
,
const
void
*
p2
)
{
TSDBKEY
*
pKey1
=
(
TSDBKEY
*
)
p1
;
...
...
source/dnode/vnode/src/tsdb/tsdbCache.c
浏览文件 @
89f7ad29
...
...
@@ -26,7 +26,7 @@ int32_t tsdbOpenCache(STsdb *pTsdb) {
goto
_err
;
}
taosLRUCacheSetStrictCapacity
(
pCache
,
tru
e
);
taosLRUCacheSetStrictCapacity
(
pCache
,
fals
e
);
taosThreadMutexInit
(
&
pTsdb
->
lruMutex
,
NULL
);
...
...
@@ -488,11 +488,12 @@ typedef struct {
int32_t
nFileSet
;
int32_t
iFileSet
;
SArray
*
aDFileSet
;
SDataFReader
*
pDataFReader
;
SDataFReader
*
*
pDataFReader
;
TSDBROW
row
;
SMergeTree
mergeTree
;
SMergeTree
*
pMergeTree
;
SMergeTree
mergeTree
;
SMergeTree
*
pMergeTree
;
SSttBlockLoadInfo
*
pLoadInfo
;
}
SFSLastNextRowIter
;
static
int32_t
getNextRowFromFSLast
(
void
*
iter
,
TSDBROW
**
ppRow
)
{
...
...
@@ -519,18 +520,20 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
return
code
;
}
if
(
state
->
pDataFReader
!=
NULL
)
{
tsdbDataFReaderClose
(
&
state
->
pDataFReader
);
state
->
pDataFReader
=
NULL
;
}
if
(
*
state
->
pDataFReader
==
NULL
||
(
*
state
->
pDataFReader
)
->
pSet
->
fid
!=
pFileSet
->
fid
)
{
if
(
*
state
->
pDataFReader
!=
NULL
)
{
tsdbDataFReaderClose
(
state
->
pDataFReader
);
code
=
tsdbDataFReaderOpen
(
&
state
->
pDataFReader
,
state
->
pTsdb
,
pFileSet
);
if
(
code
)
goto
_err
;
resetLastBlockLoadInfo
(
state
->
pLoadInfo
);
}
SSttBlockLoadInfo
*
pLoadInfo
=
tCreateLastBlockLoadInfo
(
state
->
pTSchema
,
NULL
,
0
);
tMergeTreeOpen
(
&
state
->
mergeTree
,
1
,
state
->
pDataFReader
,
state
->
suid
,
state
->
uid
,
code
=
tsdbDataFReaderOpen
(
state
->
pDataFReader
,
state
->
pTsdb
,
pFileSet
);
if
(
code
)
goto
_err
;
}
tMergeTreeOpen
(
&
state
->
mergeTree
,
1
,
*
state
->
pDataFReader
,
state
->
suid
,
state
->
uid
,
&
(
STimeWindow
){.
skey
=
TSKEY_MIN
,
.
ekey
=
TSKEY_MAX
},
&
(
SVersionRange
){.
minVer
=
0
,
.
maxVer
=
UINT64_MAX
},
pLoadInfo
,
tru
e
,
NULL
);
&
(
SVersionRange
){.
minVer
=
0
,
.
maxVer
=
UINT64_MAX
},
state
->
pLoadInfo
,
fals
e
,
NULL
);
state
->
pMergeTree
=
&
state
->
mergeTree
;
bool
hasVal
=
tMergeTreeNext
(
&
state
->
mergeTree
);
if
(
!
hasVal
)
{
...
...
@@ -554,10 +557,10 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
}
_err:
if
(
state
->
pDataFReader
)
{
/*
if (state->pDataFReader) {
tsdbDataFReaderClose(&state->pDataFReader);
state->pDataFReader = NULL;
}
}*/
if
(
state
->
pMergeTree
!=
NULL
)
{
tMergeTreeClose
(
state
->
pMergeTree
);
state
->
pMergeTree
=
NULL
;
...
...
@@ -575,12 +578,12 @@ int32_t clearNextRowFromFSLast(void *iter) {
if
(
!
state
)
{
return
code
;
}
/*
if (state->pDataFReader) {
tsdbDataFReaderClose(&state->pDataFReader);
state->pDataFReader = NULL;
}
*/
if
(
state
->
pMergeTree
!=
NULL
)
{
tMergeTreeClose
(
state
->
pMergeTree
);
state
->
pMergeTree
=
NULL
;
...
...
@@ -597,27 +600,28 @@ typedef enum SFSNEXTROWSTATES {
}
SFSNEXTROWSTATES
;
typedef
struct
SFSNextRowIter
{
SFSNEXTROWSTATES
state
;
// [input]
STsdb
*
pTsdb
;
// [input]
SBlockIdx
*
pBlockIdxExp
;
// [input]
STSchema
*
pTSchema
;
// [input]
tb_uid_t
suid
;
tb_uid_t
uid
;
int32_t
nFileSet
;
int32_t
iFileSet
;
SArray
*
aDFileSet
;
SDataFReader
*
pDataFReader
;
SArray
*
aBlockIdx
;
SBlockIdx
*
pBlockIdx
;
SMapData
blockMap
;
int32_t
nBlock
;
int32_t
iBlock
;
SDataBlk
block
;
SBlockData
blockData
;
SBlockData
*
pBlockData
;
int32_t
nRow
;
int32_t
iRow
;
TSDBROW
row
;
SFSNEXTROWSTATES
state
;
// [input]
STsdb
*
pTsdb
;
// [input]
SBlockIdx
*
pBlockIdxExp
;
// [input]
STSchema
*
pTSchema
;
// [input]
tb_uid_t
suid
;
tb_uid_t
uid
;
int32_t
nFileSet
;
int32_t
iFileSet
;
SArray
*
aDFileSet
;
SDataFReader
**
pDataFReader
;
SArray
*
aBlockIdx
;
SBlockIdx
*
pBlockIdx
;
SMapData
blockMap
;
int32_t
nBlock
;
int32_t
iBlock
;
SDataBlk
block
;
SBlockData
blockData
;
SBlockData
*
pBlockData
;
int32_t
nRow
;
int32_t
iRow
;
TSDBROW
row
;
SSttBlockLoadInfo
*
pLoadInfo
;
}
SFSNextRowIter
;
static
int32_t
getNextRowFromFS
(
void
*
iter
,
TSDBROW
**
ppRow
)
{
...
...
@@ -648,8 +652,16 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
return
code
;
}
code
=
tsdbDataFReaderOpen
(
&
state
->
pDataFReader
,
state
->
pTsdb
,
pFileSet
);
if
(
code
)
goto
_err
;
if
(
*
state
->
pDataFReader
==
NULL
||
(
*
state
->
pDataFReader
)
->
pSet
->
fid
!=
pFileSet
->
fid
)
{
if
(
*
state
->
pDataFReader
!=
NULL
)
{
tsdbDataFReaderClose
(
state
->
pDataFReader
);
resetLastBlockLoadInfo
(
state
->
pLoadInfo
);
}
code
=
tsdbDataFReaderOpen
(
state
->
pDataFReader
,
state
->
pTsdb
,
pFileSet
);
if
(
code
)
goto
_err
;
}
// tMapDataReset(&state->blockIdxMap);
if
(
!
state
->
aBlockIdx
)
{
...
...
@@ -657,7 +669,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
}
else
{
taosArrayClear
(
state
->
aBlockIdx
);
}
code
=
tsdbReadBlockIdx
(
state
->
pDataFReader
,
state
->
aBlockIdx
);
code
=
tsdbReadBlockIdx
(
*
state
->
pDataFReader
,
state
->
aBlockIdx
);
if
(
code
)
goto
_err
;
/* if (state->pBlockIdx) { */
...
...
@@ -666,17 +678,20 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
* &state->blockIdx);
*/
state
->
pBlockIdx
=
taosArraySearch
(
state
->
aBlockIdx
,
state
->
pBlockIdxExp
,
tCmprBlockIdx
,
TD_EQ
);
if
(
!
state
->
pBlockIdx
)
{
tsdbDataFReaderClose
(
&
state
->
pDataFReader
);
state
->
pDataFReader
=
NULL
;
if
(
!
state
->
pBlockIdx
)
{
/*
tsdbDataFReaderClose(state->pDataFReader);
*state->pDataFReader = NULL;
resetLastBlockLoadInfo(state->pLoadInfo);*/
goto
_next_fileset
;
}
tMapDataReset
(
&
state
->
blockMap
);
/*
if (state->blockMap.pData != NULL) {
tMapDataClear(&state->blockMap);
}
code
=
tsdbReadDataBlk
(
state
->
pDataFReader
,
state
->
pBlockIdx
,
&
state
->
blockMap
);
*/
code
=
tsdbReadDataBlk
(
*
state
->
pDataFReader
,
state
->
pBlockIdx
,
&
state
->
blockMap
);
if
(
code
)
goto
_err
;
state
->
nBlock
=
state
->
blockMap
.
nItem
;
...
...
@@ -703,7 +718,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
code
=
tBlockDataInit
(
state
->
pBlockData
,
&
tid
,
state
->
pTSchema
,
NULL
,
0
);
if
(
code
)
goto
_err
;
code
=
tsdbReadDataBlock
(
state
->
pDataFReader
,
&
block
,
state
->
pBlockData
);
code
=
tsdbReadDataBlock
(
*
state
->
pDataFReader
,
&
block
,
state
->
pBlockData
);
if
(
code
)
goto
_err
;
state
->
nRow
=
state
->
blockData
.
nRow
;
...
...
@@ -719,8 +734,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
if
(
--
state
->
iRow
<
0
)
{
state
->
state
=
SFSNEXTROW_BLOCKDATA
;
if
(
--
state
->
iBlock
<
0
)
{
tsdbDataFReaderClose
(
&
state
->
pDataFReader
);
state
->
pDataFReader
=
NULL
;
tsdbDataFReaderClose
(
state
->
pDataFReader
);
*
state
->
pDataFReader
=
NULL
;
resetLastBlockLoadInfo
(
state
->
pLoadInfo
);
if
(
state
->
aBlockIdx
)
{
taosArrayDestroy
(
state
->
aBlockIdx
);
...
...
@@ -739,16 +755,17 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
}
_err:
if
(
state
->
pDataFReader
)
{
tsdbDataFReaderClose
(
&
state
->
pDataFReader
);
state
->
pDataFReader
=
NULL
;
}
/*
if (*state->pDataFReader) {
tsdbDataFReaderClose(state->pDataFReader);
*state->pDataFReader = NULL;
resetLastBlockLoadInfo(state->pLoadInfo);
}*/
if
(
state
->
aBlockIdx
)
{
taosArrayDestroy
(
state
->
aBlockIdx
);
state
->
aBlockIdx
=
NULL
;
}
if
(
state
->
pBlockData
)
{
// tBlockDataDestroy(&state->blockData, 1);
tBlockDataDestroy
(
state
->
pBlockData
,
1
);
state
->
pBlockData
=
NULL
;
}
...
...
@@ -765,11 +782,11 @@ int32_t clearNextRowFromFS(void *iter) {
if
(
!
state
)
{
return
code
;
}
/*
if (state->pDataFReader) {
tsdbDataFReaderClose(&state->pDataFReader);
state->pDataFReader = NULL;
}
}*/
if
(
state
->
aBlockIdx
)
{
taosArrayDestroy
(
state
->
aBlockIdx
);
state
->
aBlockIdx
=
NULL
;
...
...
@@ -930,25 +947,21 @@ typedef struct {
TSDBROW
memRow
,
imemRow
,
fsLastRow
,
fsRow
;
TsdbNextRowState
input
[
4
];
STsdbReadSnap
*
pReadSnap
;
STsdb
*
pTsdb
;
}
CacheNextRowIter
;
static
int32_t
nextRowIterOpen
(
CacheNextRowIter
*
pIter
,
tb_uid_t
uid
,
STsdb
*
pTsdb
,
STSchema
*
pTSchema
)
{
static
int32_t
nextRowIterOpen
(
CacheNextRowIter
*
pIter
,
tb_uid_t
uid
,
STsdb
*
pTsdb
,
STSchema
*
pTSchema
,
tb_uid_t
suid
,
SSttBlockLoadInfo
*
pLoadInfo
,
STsdbReadSnap
*
pReadSnap
,
SDataFReader
**
pDataFReader
)
{
int
code
=
0
;
tb_uid_t
suid
=
getTableSuidByUid
(
uid
,
pTsdb
);
tsdbTakeReadSnap
(
pTsdb
,
&
pIter
->
pReadSnap
,
NULL
);
STbData
*
pMem
=
NULL
;
if
(
p
Iter
->
p
ReadSnap
->
pMem
)
{
pMem
=
tsdbGetTbDataFromMemTable
(
p
Iter
->
p
ReadSnap
->
pMem
,
suid
,
uid
);
if
(
pReadSnap
->
pMem
)
{
pMem
=
tsdbGetTbDataFromMemTable
(
pReadSnap
->
pMem
,
suid
,
uid
);
}
STbData
*
pIMem
=
NULL
;
if
(
p
Iter
->
p
ReadSnap
->
pIMem
)
{
pIMem
=
tsdbGetTbDataFromMemTable
(
p
Iter
->
p
ReadSnap
->
pIMem
,
suid
,
uid
);
if
(
pReadSnap
->
pIMem
)
{
pIMem
=
tsdbGetTbDataFromMemTable
(
pReadSnap
->
pIMem
,
suid
,
uid
);
}
pIter
->
pTsdb
=
pTsdb
;
...
...
@@ -957,7 +970,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
SDelIdx
delIdx
;
SDelFile
*
pDelFile
=
p
Iter
->
p
ReadSnap
->
fs
.
pDelFile
;
SDelFile
*
pDelFile
=
pReadSnap
->
fs
.
pDelFile
;
if
(
pDelFile
)
{
SDelFReader
*
pDelFReader
;
...
...
@@ -988,18 +1001,22 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
pIter
->
fsLastState
.
state
=
(
SFSLASTNEXTROWSTATES
)
SFSNEXTROW_FS
;
pIter
->
fsLastState
.
pTsdb
=
pTsdb
;
pIter
->
fsLastState
.
aDFileSet
=
p
Iter
->
p
ReadSnap
->
fs
.
aDFileSet
;
pIter
->
fsLastState
.
aDFileSet
=
pReadSnap
->
fs
.
aDFileSet
;
pIter
->
fsLastState
.
pTSchema
=
pTSchema
;
pIter
->
fsLastState
.
suid
=
suid
;
pIter
->
fsLastState
.
uid
=
uid
;
pIter
->
fsLastState
.
pLoadInfo
=
pLoadInfo
;
pIter
->
fsLastState
.
pDataFReader
=
pDataFReader
;
pIter
->
fsState
.
state
=
SFSNEXTROW_FS
;
pIter
->
fsState
.
pTsdb
=
pTsdb
;
pIter
->
fsState
.
aDFileSet
=
p
Iter
->
p
ReadSnap
->
fs
.
aDFileSet
;
pIter
->
fsState
.
aDFileSet
=
pReadSnap
->
fs
.
aDFileSet
;
pIter
->
fsState
.
pBlockIdxExp
=
&
pIter
->
idx
;
pIter
->
fsState
.
pTSchema
=
pTSchema
;
pIter
->
fsState
.
suid
=
suid
;
pIter
->
fsState
.
uid
=
uid
;
pIter
->
fsState
.
pLoadInfo
=
pLoadInfo
;
pIter
->
fsState
.
pDataFReader
=
pDataFReader
;
pIter
->
input
[
0
]
=
(
TsdbNextRowState
){
&
pIter
->
memRow
,
true
,
false
,
&
pIter
->
memState
,
getNextRowFromMem
,
NULL
};
pIter
->
input
[
1
]
=
(
TsdbNextRowState
){
&
pIter
->
imemRow
,
true
,
false
,
&
pIter
->
imemState
,
getNextRowFromMem
,
NULL
};
...
...
@@ -1040,8 +1057,6 @@ static int32_t nextRowIterClose(CacheNextRowIter *pIter) {
taosArrayDestroy
(
pIter
->
pSkyline
);
}
tsdbUntakeReadSnap
(
pIter
->
pTsdb
,
pIter
->
pReadSnap
,
NULL
);
_err:
return
code
;
}
...
...
@@ -1119,10 +1134,10 @@ _err:
return
code
;
}
static
int32_t
mergeLastRow
(
tb_uid_t
uid
,
STsdb
*
pTsdb
,
bool
*
dup
,
SArray
**
ppColArray
)
{
static
int32_t
mergeLastRow
(
tb_uid_t
uid
,
STsdb
*
pTsdb
,
bool
*
dup
,
SArray
**
ppColArray
,
SCacheRowsReader
*
pr
)
{
int32_t
code
=
0
;
STSchema
*
pTSchema
=
metaGetTbTSchema
(
pTsdb
->
pVnode
->
pMeta
,
uid
,
-
1
,
1
);
STSchema
*
pTSchema
=
pr
->
pSchema
;
//
metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1);
int16_t
nCol
=
pTSchema
->
numOfCols
;
int16_t
iCol
=
0
;
int16_t
noneCol
=
0
;
...
...
@@ -1133,7 +1148,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo
TSKEY
lastRowTs
=
TSKEY_MAX
;
CacheNextRowIter
iter
=
{
0
};
nextRowIterOpen
(
&
iter
,
uid
,
pTsdb
,
pTSchema
);
nextRowIterOpen
(
&
iter
,
uid
,
pTsdb
,
pTSchema
,
pr
->
suid
,
pr
->
pLoadInfo
,
pr
->
pReadSnap
,
&
pr
->
pDataFReader
);
do
{
TSDBROW
*
pRow
=
NULL
;
...
...
@@ -1233,20 +1248,20 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo
}
nextRowIterClose
(
&
iter
);
taosMemoryFreeClear
(
pTSchema
);
//
taosMemoryFreeClear(pTSchema);
return
code
;
_err:
nextRowIterClose
(
&
iter
);
taosArrayDestroy
(
pColArray
);
taosMemoryFreeClear
(
pTSchema
);
//
taosMemoryFreeClear(pTSchema);
return
code
;
}
static
int32_t
mergeLast
(
tb_uid_t
uid
,
STsdb
*
pTsdb
,
SArray
**
ppLastArray
)
{
static
int32_t
mergeLast
(
tb_uid_t
uid
,
STsdb
*
pTsdb
,
SArray
**
ppLastArray
,
SCacheRowsReader
*
pr
)
{
int32_t
code
=
0
;
STSchema
*
pTSchema
=
metaGetTbTSchema
(
pTsdb
->
pVnode
->
pMeta
,
uid
,
-
1
,
1
);
STSchema
*
pTSchema
=
pr
->
pSchema
;
//
metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1);
int16_t
nCol
=
pTSchema
->
numOfCols
;
int16_t
iCol
=
0
;
int16_t
noneCol
=
0
;
...
...
@@ -1257,7 +1272,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
TSKEY
lastRowTs
=
TSKEY_MAX
;
CacheNextRowIter
iter
=
{
0
};
nextRowIterOpen
(
&
iter
,
uid
,
pTsdb
,
pTSchema
);
nextRowIterOpen
(
&
iter
,
uid
,
pTsdb
,
pTSchema
,
pr
->
suid
,
pr
->
pLoadInfo
,
pr
->
pReadSnap
,
&
pr
->
pDataFReader
);
do
{
TSDBROW
*
pRow
=
NULL
;
...
...
@@ -1350,18 +1365,18 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
}
nextRowIterClose
(
&
iter
);
taosMemoryFreeClear
(
pTSchema
);
//
taosMemoryFreeClear(pTSchema);
return
code
;
_err:
nextRowIterClose
(
&
iter
);
taosMemoryFreeClear
(
pTSchema
);
//
taosMemoryFreeClear(pTSchema);
*
ppLastArray
=
NULL
;
taosArrayDestroy
(
pColArray
);
return
code
;
}
int32_t
tsdbCacheGetLastrowH
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
S
Tsdb
*
pTsdb
,
LRUHandle
**
handle
)
{
int32_t
tsdbCacheGetLastrowH
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
S
CacheRowsReader
*
pr
,
LRUHandle
**
handle
)
{
int32_t
code
=
0
;
char
key
[
32
]
=
{
0
};
int
keyLen
=
0
;
...
...
@@ -1370,13 +1385,14 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUH
getTableCacheKey
(
uid
,
0
,
key
,
&
keyLen
);
LRUHandle
*
h
=
taosLRUCacheLookup
(
pCache
,
key
,
keyLen
);
if
(
!
h
)
{
STsdb
*
pTsdb
=
pr
->
pVnode
->
pTsdb
;
taosThreadMutexLock
(
&
pTsdb
->
lruMutex
);
h
=
taosLRUCacheLookup
(
pCache
,
key
,
keyLen
);
if
(
!
h
)
{
SArray
*
pArray
=
NULL
;
bool
dup
=
false
;
// which is always false for now
code
=
mergeLastRow
(
uid
,
pTsdb
,
&
dup
,
&
pArray
);
code
=
mergeLastRow
(
uid
,
pTsdb
,
&
dup
,
&
pArray
,
pr
);
// if table's empty or error, return code of -1
if
(
code
<
0
||
pArray
==
NULL
)
{
if
(
!
dup
&&
pArray
)
{
...
...
@@ -1392,17 +1408,17 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUH
size_t
charge
=
pArray
->
capacity
*
pArray
->
elemSize
+
sizeof
(
*
pArray
);
_taos_lru_deleter_t
deleter
=
deleteTableCacheLast
;
LRUStatus
status
=
taosLRUCacheInsert
(
pCache
,
key
,
keyLen
,
pArray
,
charge
,
deleter
,
NULL
,
TAOS_LRU_PRIORITY_LOW
);
LRUStatus
status
=
taosLRUCacheInsert
(
pCache
,
key
,
keyLen
,
pArray
,
charge
,
deleter
,
&
h
,
TAOS_LRU_PRIORITY_LOW
);
if
(
status
!=
TAOS_LRU_STATUS_OK
)
{
code
=
-
1
;
}
taosThreadMutexUnlock
(
&
pTsdb
->
lruMutex
);
//
taosThreadMutexUnlock(&pTsdb->lruMutex);
h
=
taosLRUCacheLookup
(
pCache
,
key
,
keyLen
);
}
else
{
taosThreadMutexUnlock
(
&
pTsdb
->
lruMutex
);
}
//
h = taosLRUCacheLookup(pCache, key, keyLen);
}
//
else {
taosThreadMutexUnlock
(
&
pTsdb
->
lruMutex
);
//
}
}
*
handle
=
h
;
...
...
@@ -1434,7 +1450,7 @@ _err:
return
code
;
}
int32_t
tsdbCacheGetLastH
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
S
Tsdb
*
pTsdb
,
LRUHandle
**
handle
)
{
int32_t
tsdbCacheGetLastH
(
SLRUCache
*
pCache
,
tb_uid_t
uid
,
S
CacheRowsReader
*
pr
,
LRUHandle
**
handle
)
{
int32_t
code
=
0
;
char
key
[
32
]
=
{
0
};
int
keyLen
=
0
;
...
...
@@ -1443,12 +1459,13 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand
getTableCacheKey
(
uid
,
1
,
key
,
&
keyLen
);
LRUHandle
*
h
=
taosLRUCacheLookup
(
pCache
,
key
,
keyLen
);
if
(
!
h
)
{
STsdb
*
pTsdb
=
pr
->
pVnode
->
pTsdb
;
taosThreadMutexLock
(
&
pTsdb
->
lruMutex
);
h
=
taosLRUCacheLookup
(
pCache
,
key
,
keyLen
);
if
(
!
h
)
{
SArray
*
pLastArray
=
NULL
;
code
=
mergeLast
(
uid
,
pTsdb
,
&
pLastArray
);
code
=
mergeLast
(
uid
,
pTsdb
,
&
pLastArray
,
pr
);
// if table's empty or error, return code of -1
if
(
code
<
0
||
pLastArray
==
NULL
)
{
taosThreadMutexUnlock
(
&
pTsdb
->
lruMutex
);
...
...
@@ -1460,17 +1477,17 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand
size_t
charge
=
pLastArray
->
capacity
*
pLastArray
->
elemSize
+
sizeof
(
*
pLastArray
);
_taos_lru_deleter_t
deleter
=
deleteTableCacheLast
;
LRUStatus
status
=
taosLRUCacheInsert
(
pCache
,
key
,
keyLen
,
pLastArray
,
charge
,
deleter
,
NULL
,
TAOS_LRU_PRIORITY_LOW
);
taosLRUCacheInsert
(
pCache
,
key
,
keyLen
,
pLastArray
,
charge
,
deleter
,
&
h
,
TAOS_LRU_PRIORITY_LOW
);
if
(
status
!=
TAOS_LRU_STATUS_OK
)
{
code
=
-
1
;
}
taosThreadMutexUnlock
(
&
pTsdb
->
lruMutex
);
//
taosThreadMutexUnlock(&pTsdb->lruMutex);
h
=
taosLRUCacheLookup
(
pCache
,
key
,
keyLen
);
}
else
{
taosThreadMutexUnlock
(
&
pTsdb
->
lruMutex
);
}
//
h = taosLRUCacheLookup(pCache, key, keyLen);
}
//
else {
taosThreadMutexUnlock
(
&
pTsdb
->
lruMutex
);
//
}
}
*
handle
=
h
;
...
...
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
浏览文件 @
89f7ad29
...
...
@@ -18,18 +18,7 @@
#include "tcommon.h"
#include "tsdb.h"
typedef
struct
SCacheRowsReader
{
SVnode
*
pVnode
;
STSchema
*
pSchema
;
uint64_t
uid
;
char
**
transferBuf
;
// todo remove it soon
int32_t
numOfCols
;
int32_t
type
;
int32_t
tableIndex
;
// currently returned result tables
SArray
*
pTableList
;
// table id list
}
SCacheRowsReader
;
#define HASTYPE(_type, _t) (((_type) & (_t)) == (_t))
#define HASTYPE(_type, _t) (((_type) & (_t)) == (_t))
static
void
saveOneRow
(
SArray
*
pRow
,
SSDataBlock
*
pBlock
,
SCacheRowsReader
*
pReader
,
const
int32_t
*
slotIds
,
void
**
pRes
)
{
...
...
@@ -56,7 +45,7 @@ static void saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* pRea
if
(
IS_VAR_DATA_TYPE
(
pColVal
->
colVal
.
type
))
{
varDataSetLen
(
p
->
buf
,
pColVal
->
colVal
.
value
.
nData
);
memcpy
(
varDataVal
(
p
->
buf
),
pColVal
->
colVal
.
value
.
pData
,
pColVal
->
colVal
.
value
.
nData
);
p
->
bytes
=
pColVal
->
colVal
.
value
.
nData
+
VARSTR_HEADER_SIZE
;
// binary needs to plus the header size
p
->
bytes
=
pColVal
->
colVal
.
value
.
nData
+
VARSTR_HEADER_SIZE
;
// binary needs to plus the header size
}
else
{
memcpy
(
p
->
buf
,
&
pColVal
->
colVal
.
value
,
pReader
->
pSchema
->
columns
[
slotId
].
bytes
);
p
->
bytes
=
pReader
->
pSchema
->
columns
[
slotId
].
bytes
;
...
...
@@ -101,7 +90,8 @@ static void saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* pRea
pBlock
->
info
.
rows
+=
1
;
}
int32_t
tsdbCacherowsReaderOpen
(
void
*
pVnode
,
int32_t
type
,
SArray
*
pTableIdList
,
int32_t
numOfCols
,
void
**
pReader
)
{
int32_t
tsdbCacherowsReaderOpen
(
void
*
pVnode
,
int32_t
type
,
SArray
*
pTableIdList
,
int32_t
numOfCols
,
uint64_t
suid
,
void
**
pReader
)
{
*
pReader
=
NULL
;
SCacheRowsReader
*
p
=
taosMemoryCalloc
(
1
,
sizeof
(
SCacheRowsReader
));
...
...
@@ -112,6 +102,7 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, SArray* pTableIdList
p
->
type
=
type
;
p
->
pVnode
=
pVnode
;
p
->
numOfCols
=
numOfCols
;
p
->
suid
=
suid
;
if
(
taosArrayGetSize
(
pTableIdList
)
==
0
)
{
*
pReader
=
p
;
...
...
@@ -138,6 +129,12 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, SArray* pTableIdList
}
}
p
->
pLoadInfo
=
tCreateLastBlockLoadInfo
(
p
->
pSchema
,
NULL
,
0
);
if
(
p
->
pLoadInfo
==
NULL
)
{
tsdbCacherowsReaderClose
(
p
);
return
TSDB_CODE_OUT_OF_MEMORY
;
}
*
pReader
=
p
;
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -154,6 +151,8 @@ void* tsdbCacherowsReaderClose(void* pReader) {
taosMemoryFree
(
p
->
pSchema
);
}
destroyLastBlockLoadInfo
(
p
->
pLoadInfo
);
taosMemoryFree
(
pReader
);
return
NULL
;
}
...
...
@@ -164,9 +163,9 @@ static int32_t doExtractCacheRow(SCacheRowsReader* pr, SLRUCache* lruCache, uint
*
pRow
=
NULL
;
if
(
HASTYPE
(
pr
->
type
,
CACHESCAN_RETRIEVE_LAST_ROW
))
{
code
=
tsdbCacheGetLastrowH
(
lruCache
,
uid
,
pr
->
pVnode
->
pTsdb
,
h
);
code
=
tsdbCacheGetLastrowH
(
lruCache
,
uid
,
pr
,
h
);
}
else
{
code
=
tsdbCacheGetLastH
(
lruCache
,
uid
,
pr
->
pVnode
->
pTsdb
,
h
);
code
=
tsdbCacheGetLastH
(
lruCache
,
uid
,
pr
,
h
);
}
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
...
...
@@ -182,7 +181,7 @@ static int32_t doExtractCacheRow(SCacheRowsReader* pr, SLRUCache* lruCache, uint
}
static
void
freeItem
(
void
*
pItem
)
{
SLastCol
*
pCol
=
(
SLastCol
*
)
pItem
;
SLastCol
*
pCol
=
(
SLastCol
*
)
pItem
;
if
(
IS_VAR_DATA_TYPE
(
pCol
->
colVal
.
type
))
{
taosMemoryFree
(
pCol
->
colVal
.
value
.
pData
);
}
...
...
@@ -223,7 +222,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
for
(
int32_t
i
=
0
;
i
<
pr
->
pSchema
->
numOfCols
;
++
i
)
{
struct
STColumn
*
pCol
=
&
pr
->
pSchema
->
columns
[
i
];
SLastCol
p
=
{.
ts
=
INT64_MIN
,
.
colVal
.
type
=
pCol
->
type
};
SLastCol
p
=
{.
ts
=
INT64_MIN
,
.
colVal
.
type
=
pCol
->
type
};
if
(
IS_VAR_DATA_TYPE
(
pCol
->
type
))
{
p
.
colVal
.
value
.
pData
=
taosMemoryCalloc
(
pCol
->
bytes
,
sizeof
(
char
));
...
...
@@ -231,6 +230,9 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
taosArrayPush
(
pLastCols
,
&
p
);
}
tsdbTakeReadSnap
(
pr
->
pVnode
->
pTsdb
,
&
pr
->
pReadSnap
,
"cache-l"
);
pr
->
pDataFReader
=
NULL
;
// retrieve the only one last row of all tables in the uid list.
if
(
HASTYPE
(
pr
->
type
,
CACHESCAN_RETRIEVE_TYPE_SINGLE
))
{
for
(
int32_t
i
=
0
;
i
<
numOfTables
;
++
i
)
{
...
...
@@ -299,7 +301,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
}
else
if
(
HASTYPE
(
pr
->
type
,
CACHESCAN_RETRIEVE_TYPE_ALL
))
{
for
(
int32_t
i
=
pr
->
tableIndex
;
i
<
numOfTables
;
++
i
)
{
STableKeyInfo
*
pKeyInfo
=
(
STableKeyInfo
*
)
taosArrayGet
(
pr
->
pTableList
,
i
);
STableKeyInfo
*
pKeyInfo
=
(
STableKeyInfo
*
)
taosArrayGet
(
pr
->
pTableList
,
i
);
code
=
doExtractCacheRow
(
pr
,
lruCache
,
pKeyInfo
->
uid
,
&
pRow
,
&
h
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
return
code
;
...
...
@@ -324,7 +326,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
code
=
TSDB_CODE_INVALID_PARA
;
}
_end:
_end:
tsdbDataFReaderClose
(
&
pr
->
pDataFReader
);
tsdbUntakeReadSnap
(
pr
->
pVnode
->
pTsdb
,
pr
->
pReadSnap
,
"cache-l"
);
for
(
int32_t
j
=
0
;
j
<
pr
->
numOfCols
;
++
j
)
{
taosMemoryFree
(
pRes
[
j
]);
}
...
...
source/libs/executor/src/cachescanoperator.c
浏览文件 @
89f7ad29
...
...
@@ -58,17 +58,19 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
// partition by tbname
if
(
taosArrayGetSize
(
pTableList
->
pGroupList
)
==
taosArrayGetSize
(
pTableList
->
pTableList
))
{
pInfo
->
retrieveType
=
CACHESCAN_RETRIEVE_TYPE_ALL
|
(
pScanNode
->
ignoreNull
?
CACHESCAN_RETRIEVE_LAST
:
CACHESCAN_RETRIEVE_LAST_ROW
);
pInfo
->
retrieveType
=
CACHESCAN_RETRIEVE_TYPE_ALL
|
(
pScanNode
->
ignoreNull
?
CACHESCAN_RETRIEVE_LAST
:
CACHESCAN_RETRIEVE_LAST_ROW
);
code
=
tsdbCacherowsReaderOpen
(
pInfo
->
readHandle
.
vnode
,
pInfo
->
retrieveType
,
pTableList
->
pTableList
,
taosArrayGetSize
(
pInfo
->
pColMatchInfo
),
&
pInfo
->
pLastrowReader
);
taosArrayGetSize
(
pInfo
->
pColMatchInfo
),
pTableList
->
suid
,
&
pInfo
->
pLastrowReader
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
goto
_error
;
}
pInfo
->
pBufferredRes
=
createOneDataBlock
(
pInfo
->
pRes
,
false
);
blockDataEnsureCapacity
(
pInfo
->
pBufferredRes
,
pOperator
->
resultInfo
.
capacity
);
}
else
{
// by tags
pInfo
->
retrieveType
=
CACHESCAN_RETRIEVE_TYPE_SINGLE
|
(
pScanNode
->
ignoreNull
?
CACHESCAN_RETRIEVE_LAST
:
CACHESCAN_RETRIEVE_LAST_ROW
);
}
else
{
// by tags
pInfo
->
retrieveType
=
CACHESCAN_RETRIEVE_TYPE_SINGLE
|
(
pScanNode
->
ignoreNull
?
CACHESCAN_RETRIEVE_LAST
:
CACHESCAN_RETRIEVE_LAST_ROW
);
}
if
(
pScanNode
->
scan
.
pScanPseudoCols
!=
NULL
)
{
...
...
@@ -184,7 +186,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
SArray
*
pGroupTableList
=
taosArrayGetP
(
pTableList
->
pGroupList
,
pInfo
->
currentGroupIndex
);
tsdbCacherowsReaderOpen
(
pInfo
->
readHandle
.
vnode
,
pInfo
->
retrieveType
,
pGroupTableList
,
taosArrayGetSize
(
pInfo
->
pColMatchInfo
),
&
pInfo
->
pLastrowReader
);
taosArrayGetSize
(
pInfo
->
pColMatchInfo
),
pTableList
->
suid
,
&
pInfo
->
pLastrowReader
);
taosArrayClear
(
pInfo
->
pUidList
);
int32_t
code
=
tsdbRetrieveCacheRows
(
pInfo
->
pLastrowReader
,
pInfo
->
pRes
,
pInfo
->
pSlotIds
,
pInfo
->
pUidList
);
...
...
@@ -265,4 +267,4 @@ int32_t extractTargetSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInf
}
return
TSDB_CODE_SUCCESS
;
}
\ No newline at end of file
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录