Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
4bc8e086
T
TDengine
项目概览
taosdata
/
TDengine
大约 1 年 前同步成功
通知
1185
Star
22015
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
4bc8e086
编写于
8月 21, 2022
作者:
S
Shengliang Guan
提交者:
GitHub
8月 21, 2022
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #16262 from taosdata/feature/3.0_interval_hash_optimize
enh: rsma batch process and code optimization
上级
99b49877
e0317964
变更
17
展开全部
隐藏空白更改
内联
并排
Showing
17 changed file
with
324 addition
and
392 deletion
+324
-392
include/common/tmsg.h
include/common/tmsg.h
+0
-25
include/common/tmsgdef.h
include/common/tmsgdef.h
+1
-1
include/util/tqueue.h
include/util/tqueue.h
+1
-0
source/dnode/mnode/impl/src/mndStb.c
source/dnode/mnode/impl/src/mndStb.c
+2
-0
source/dnode/vnode/src/inc/sma.h
source/dnode/vnode/src/inc/sma.h
+14
-11
source/dnode/vnode/src/inc/vnodeInt.h
source/dnode/vnode/src/inc/vnodeInt.h
+2
-1
source/dnode/vnode/src/sma/smaCommit.c
source/dnode/vnode/src/sma/smaCommit.c
+33
-38
source/dnode/vnode/src/sma/smaEnv.c
source/dnode/vnode/src/sma/smaEnv.c
+5
-12
source/dnode/vnode/src/sma/smaOpen.c
source/dnode/vnode/src/sma/smaOpen.c
+14
-0
source/dnode/vnode/src/sma/smaRollup.c
source/dnode/vnode/src/sma/smaRollup.c
+219
-292
source/dnode/vnode/src/sma/smaUtil.c
source/dnode/vnode/src/sma/smaUtil.c
+3
-0
source/dnode/vnode/src/vnd/vnodeCommit.c
source/dnode/vnode/src/vnd/vnodeCommit.c
+18
-9
source/dnode/vnode/src/vnd/vnodeOpen.c
source/dnode/vnode/src/vnd/vnodeOpen.c
+3
-0
source/dnode/vnode/src/vnd/vnodeSvr.c
source/dnode/vnode/src/vnd/vnodeSvr.c
+0
-2
source/libs/executor/src/executorimpl.c
source/libs/executor/src/executorimpl.c
+1
-0
source/util/src/tqueue.c
source/util/src/tqueue.c
+2
-1
tests/script/tsim/sma/rsmaPersistenceRecovery.sim
tests/script/tsim/sma/rsmaPersistenceRecovery.sim
+6
-0
未找到文件。
include/common/tmsg.h
浏览文件 @
4bc8e086
...
...
@@ -2667,31 +2667,6 @@ typedef struct {
int32_t
padding
;
}
SRSmaExecMsg
;
typedef
struct
{
int64_t
suid
;
int8_t
level
;
}
SRSmaFetchMsg
;
static
FORCE_INLINE
int32_t
tEncodeSRSmaFetchMsg
(
SEncoder
*
pCoder
,
const
SRSmaFetchMsg
*
pReq
)
{
if
(
tStartEncode
(
pCoder
)
<
0
)
return
-
1
;
if
(
tEncodeI64
(
pCoder
,
pReq
->
suid
)
<
0
)
return
-
1
;
if
(
tEncodeI8
(
pCoder
,
pReq
->
level
)
<
0
)
return
-
1
;
tEndEncode
(
pCoder
);
return
0
;
}
static
FORCE_INLINE
int32_t
tDecodeSRSmaFetchMsg
(
SDecoder
*
pCoder
,
SRSmaFetchMsg
*
pReq
)
{
if
(
tStartDecode
(
pCoder
)
<
0
)
return
-
1
;
if
(
tDecodeI64
(
pCoder
,
&
pReq
->
suid
)
<
0
)
return
-
1
;
if
(
tDecodeI8
(
pCoder
,
&
pReq
->
level
)
<
0
)
return
-
1
;
tEndDecode
(
pCoder
);
return
0
;
}
typedef
struct
{
int8_t
version
;
// for compatibility(default 0)
int8_t
intervalUnit
;
// MACRO: TIME_UNIT_XXX
...
...
include/common/tmsgdef.h
浏览文件 @
4bc8e086
...
...
@@ -201,7 +201,7 @@ enum {
TD_DEF_MSG_TYPE
(
TDMT_VND_CANCEL_SMA
,
"vnode-cancel-sma"
,
NULL
,
NULL
)
TD_DEF_MSG_TYPE
(
TDMT_VND_DROP_SMA
,
"vnode-drop-sma"
,
NULL
,
NULL
)
TD_DEF_MSG_TYPE
(
TDMT_VND_SUBMIT_RSMA
,
"vnode-submit-rsma"
,
SSubmitReq
,
SSubmitRsp
)
TD_DEF_MSG_TYPE
(
TDMT_VND_FETCH_RSMA
,
"vnode-fetch-rsma"
,
SRSmaFetchMsg
,
NULL
)
TD_DEF_MSG_TYPE
(
TDMT_VND_FETCH_RSMA
,
"vnode-fetch-rsma"
,
NULL
,
NULL
)
TD_DEF_MSG_TYPE
(
TDMT_VND_EXEC_RSMA
,
"vnode-exec-rsma"
,
NULL
,
NULL
)
TD_DEF_MSG_TYPE
(
TDMT_VND_DELETE
,
"delete-data"
,
SVDeleteReq
,
SVDeleteRsp
)
TD_DEF_MSG_TYPE
(
TDMT_VND_BATCH_DEL
,
"batch-delete"
,
SBatchDeleteReq
,
NULL
)
...
...
include/util/tqueue.h
浏览文件 @
4bc8e086
...
...
@@ -76,6 +76,7 @@ void taosFreeQall(STaosQall *qall);
int32_t
taosReadAllQitems
(
STaosQueue
*
queue
,
STaosQall
*
qall
);
int32_t
taosGetQitem
(
STaosQall
*
qall
,
void
**
ppItem
);
void
taosResetQitems
(
STaosQall
*
qall
);
int32_t
taosQallItemSize
(
STaosQall
*
qall
);
STaosQset
*
taosOpenQset
();
void
taosCloseQset
(
STaosQset
*
qset
);
...
...
source/dnode/mnode/impl/src/mndStb.c
浏览文件 @
4bc8e086
...
...
@@ -442,6 +442,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
if
(
req
.
rollup
)
{
req
.
rsmaParam
.
maxdelay
[
0
]
=
pStb
->
maxdelay
[
0
];
req
.
rsmaParam
.
maxdelay
[
1
]
=
pStb
->
maxdelay
[
1
];
req
.
rsmaParam
.
watermark
[
0
]
=
pStb
->
watermark
[
0
];
req
.
rsmaParam
.
watermark
[
1
]
=
pStb
->
watermark
[
1
];
if
(
pStb
->
ast1Len
>
0
)
{
if
(
mndConvertRsmaTask
(
&
req
.
rsmaParam
.
qmsg
[
0
],
&
req
.
rsmaParam
.
qmsgLen
[
0
],
pStb
->
pAst1
,
pStb
->
uid
,
STREAM_TRIGGER_WINDOW_CLOSE
,
req
.
rsmaParam
.
watermark
[
0
])
<
0
)
{
...
...
source/dnode/vnode/src/inc/sma.h
浏览文件 @
4bc8e086
...
...
@@ -32,7 +32,8 @@ extern "C" {
#define smaTrace(...) do { if (smaDebugFlag & DEBUG_TRACE) { taosPrintLog("SMA ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
// clang-format on
#define RSMA_TASK_INFO_HASH_SLOT 8
#define RSMA_TASK_INFO_HASH_SLOT (8)
#define RSMA_EXECUTOR_MAX (1)
typedef
struct
SSmaEnv
SSmaEnv
;
typedef
struct
SSmaStat
SSmaStat
;
...
...
@@ -90,14 +91,14 @@ struct SRSmaStat {
SSma
*
pSma
;
int64_t
commitAppliedVer
;
// vnode applied version for async commit
int64_t
refId
;
// shared by fetch tasks
volatile
int64_t
qBufSize
;
// queue buffer size
volatile
int64_t
nBufItems
;
// number of items in queue buffer
SRWLatch
lock
;
// r/w lock for rsma fs(e.g. qtaskinfo)
volatile
int8_t
nExecutor
;
// [1, max(half of query threads, 4)]
int8_t
triggerStat
;
// shared by fetch tasks
int8_t
commitStat
;
// 0 not in committing, 1 in committing
int8_t
execStat
;
// 0 not in exec , 1 in exec
SArray
*
aTaskFile
;
// qTaskFiles committed recently(for recovery/snapshot r/w)
SHashObj
*
infoHash
;
// key: suid, value: SRSmaInfo
SHashObj
*
fetchHash
;
// key: suid, value: L1 or L2 or L1|L2
tsem_t
notEmpty
;
// has items in queue buffer
};
struct
SSmaStat
{
...
...
@@ -111,26 +112,28 @@ struct SSmaStat {
#define SMA_STAT_TSMA(s) (&(s)->tsmaStat)
#define SMA_STAT_RSMA(s) (&(s)->rsmaStat)
#define RSMA_INFO_HASH(r) ((r)->infoHash)
#define RSMA_FETCH_HASH(r) ((r)->fetchHash)
#define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat)
#define RSMA_COMMIT_STAT(r) (&(r)->commitStat)
#define RSMA_REF_ID(r) ((r)->refId)
#define RSMA_FS_LOCK(r) (&(r)->lock)
struct
SRSmaInfoItem
{
int8_t
level
;
int8_t
level
:
4
;
int8_t
fetchLevel
:
4
;
int8_t
triggerStat
;
uint16_t
interval
;
// second
int32_t
maxDelay
;
uint16_t
nSkipped
;
int32_t
maxDelay
;
// ms
tmr_h
tmrId
;
};
struct
SRSmaInfo
{
STSchema
*
pTSchema
;
int64_t
suid
;
int64_t
refId
;
// refId of SRSmaStat
uint64_t
delFlag
:
1
;
uint64_t
lastReceived
:
63
;
// second
int64_t
refId
;
// refId of SRSmaStat
int64_t
lastRecv
;
// ms
int8_t
assigned
;
// 0 idle, 1 assgined for exec
int8_t
delFlag
;
int16_t
padding
;
T_REF_DECLARE
()
SRSmaInfoItem
items
[
TSDB_RETENTION_L2
];
void
*
taskInfo
[
TSDB_RETENTION_L2
];
// qTaskInfo_t
...
...
source/dnode/vnode/src/inc/vnodeInt.h
浏览文件 @
4bc8e086
...
...
@@ -189,6 +189,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
int32_t
smaInit
();
void
smaCleanUp
();
int32_t
smaOpen
(
SVnode
*
pVnode
);
int32_t
smaPreClose
(
SSma
*
pSma
);
int32_t
smaClose
(
SSma
*
pSma
);
int32_t
smaBegin
(
SSma
*
pSma
);
int32_t
smaSyncPreCommit
(
SSma
*
pSma
);
...
...
@@ -198,7 +199,6 @@ int32_t smaAsyncPreCommit(SSma* pSma);
int32_t
smaAsyncCommit
(
SSma
*
pSma
);
int32_t
smaAsyncPostCommit
(
SSma
*
pSma
);
int32_t
smaDoRetention
(
SSma
*
pSma
,
int64_t
now
);
int32_t
smaProcessFetch
(
SSma
*
pSma
,
void
*
pMsg
);
int32_t
smaProcessExec
(
SSma
*
pSma
,
void
*
pMsg
);
int32_t
tdProcessTSmaCreate
(
SSma
*
pSma
,
int64_t
version
,
const
char
*
msg
);
...
...
@@ -323,6 +323,7 @@ struct SVnode {
TdThreadMutex
lock
;
bool
blocked
;
bool
restored
;
bool
inClose
;
tsem_t
syncSem
;
SQHandle
*
pQuery
;
};
...
...
source/dnode/vnode/src/sma/smaCommit.c
浏览文件 @
4bc8e086
...
...
@@ -109,7 +109,7 @@ int32_t smaBegin(SSma *pSma) {
/**
* @brief pre-commit for rollup sma(sync commit).
* 1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED.
* 2) wait
all triggered fetch tasks finished
* 2) wait
for all triggered fetch tasks to finish
* 3) perform persist task for qTaskInfo
*
* @param pSma
...
...
@@ -127,14 +127,14 @@ static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma) {
// step 1: set rsma stat paused
atomic_store_8
(
RSMA_TRIGGER_STAT
(
pRSmaStat
),
TASK_TRIGGER_STAT_PAUSED
);
// step 2: wait
all triggered fetch tasks finished
// step 2: wait
for all triggered fetch tasks to finish
int32_t
nLoops
=
0
;
while
(
1
)
{
if
(
T_REF_VAL_GET
(
pStat
)
==
0
)
{
smaDebug
(
"vgId:%d, rsma fetch tasks all finished"
,
SMA_VID
(
pSma
));
smaDebug
(
"vgId:%d, rsma fetch tasks a
re a
ll finished"
,
SMA_VID
(
pSma
));
break
;
}
else
{
smaDebug
(
"vgId:%d, rsma fetch tasks not all finished yet"
,
SMA_VID
(
pSma
));
smaDebug
(
"vgId:%d, rsma fetch tasks
are
not all finished yet"
,
SMA_VID
(
pSma
));
}
++
nLoops
;
if
(
nLoops
>
1000
)
{
...
...
@@ -316,15 +316,17 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
// step 1: set rsma stat
atomic_store_8
(
RSMA_TRIGGER_STAT
(
pRSmaStat
),
TASK_TRIGGER_STAT_PAUSED
);
atomic_store_8
(
RSMA_COMMIT_STAT
(
pRSmaStat
),
1
);
pRSmaStat
->
commitAppliedVer
=
pSma
->
pVnode
->
state
.
applied
;
ASSERT
(
pRSmaStat
->
commitAppliedVer
>
0
);
// step 2: wait
all triggered fetch tasks finished
// step 2: wait
for all triggered fetch tasks to finish
int32_t
nLoops
=
0
;
while
(
1
)
{
if
(
T_REF_VAL_GET
(
pStat
)
==
0
)
{
smaDebug
(
"vgId:%d, rsma
fetch tasks
all finished"
,
SMA_VID
(
pSma
));
smaDebug
(
"vgId:%d, rsma
commit, fetch tasks are
all finished"
,
SMA_VID
(
pSma
));
break
;
}
else
{
smaDebug
(
"vgId:%d, rsma
fetch tasks
not all finished yet"
,
SMA_VID
(
pSma
));
smaDebug
(
"vgId:%d, rsma
commit, fetch tasks are
not all finished yet"
,
SMA_VID
(
pSma
));
}
++
nLoops
;
if
(
nLoops
>
1000
)
{
...
...
@@ -338,30 +340,29 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
* 1) This is high cost task and should not put in asyncPreCommit originally.
* 2) But, if put in asyncCommit, would trigger taskInfo cloning frequently.
*/
nLoops
=
0
;
smaInfo
(
"vgId:%d, start to wait for rsma qtask free, TID:%p"
,
SMA_VID
(
pSma
),
(
void
*
)
taosGetSelfPthreadId
());
if
(
tdRSmaProcessExecImpl
(
pSma
,
RSMA_EXEC_COMMIT
)
<
0
)
{
return
TSDB_CODE_FAILED
;
}
int8_t
old
;
while
(
1
)
{
old
=
atomic_val_compare_exchange_8
(
&
pRSmaStat
->
execStat
,
0
,
1
);
if
(
old
==
0
)
break
;
if
(
++
nLoops
>
1000
)
{
smaInfo
(
"vgId:%d, rsma commit, wait for all items to be consumed, TID:%p"
,
SMA_VID
(
pSma
),
(
void
*
)
taosGetSelfPthreadId
())
;
nLoops
=
0
;
while
(
atomic_load_64
(
&
pRSmaStat
->
nBufItems
)
>
0
)
{
++
nLoops
;
if
(
nLoops
>
1000
)
{
sched_yield
();
nLoops
=
0
;
smaDebug
(
"vgId:%d, wait for rsma qtask free, TID:%p"
,
SMA_VID
(
pSma
),
(
void
*
)
taosGetSelfPthreadId
());
}
}
smaInfo
(
"vgId:%d, end to wait for rsma qtask free, TID:%p"
,
SMA_VID
(
pSma
),
(
void
*
)
taosGetSelfPthreadId
());
if
(
tdRSmaProcessExecImpl
(
pSma
,
RSMA_EXEC_COMMIT
)
<
0
)
{
atomic_store_8
(
&
pRSmaStat
->
execStat
,
0
);
smaInfo
(
"vgId:%d, rsma commit, all items are consumed, TID:%p"
,
SMA_VID
(
pSma
),
(
void
*
)
taosGetSelfPthreadId
());
if
(
tdRSmaPersistExecImpl
(
pRSmaStat
,
RSMA_INFO_HASH
(
pRSmaStat
))
<
0
)
{
return
TSDB_CODE_FAILED
;
}
smaInfo
(
"vgId:%d, rsma commit, operator state commited, TID:%p"
,
SMA_VID
(
pSma
),
(
void
*
)
taosGetSelfPthreadId
());
#if 0 // consuming task of qTaskInfo clone
// step 4: swap queue/qall and iQueue/iQall
// lock
taosWLockLatch
(
SMA_ENV_LOCK
(
pEnv
));
//
taosWLockLatch(SMA_ENV_LOCK(pEnv));
ASSERT(RSMA_INFO_HASH(pRSmaStat));
...
...
@@ -376,13 +377,9 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter);
}
atomic_store_64
(
&
pRSmaStat
->
qBufSize
,
0
);
atomic_store_8
(
&
pRSmaStat
->
execStat
,
0
);
// unlock
taosWUnLockLatch
(
SMA_ENV_LOCK
(
pEnv
));
// step 5: others
pRSmaStat
->
commitAppliedVer
=
pSma
->
pVnode
->
state
.
applied
;
// taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
#endif
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -398,13 +395,14 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) {
if
(
!
pSmaEnv
)
{
return
TSDB_CODE_SUCCESS
;
}
#if 0
SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv);
// perform persist task for qTaskInfo operator
if (tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)) < 0) {
return TSDB_CODE_FAILED;
}
#endif
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -426,10 +424,10 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
// step 1: merge qTaskInfo and iQTaskInfo
// lock
taosWLockLatch
(
SMA_ENV_LOCK
(
pEnv
));
//
taosWLockLatch(SMA_ENV_LOCK(pEnv));
void
*
pIter
=
taosHashIterate
(
RSMA_INFO_HASH
(
pRSmaStat
),
NULL
)
;
while
(
pIter
)
{
void
*
pIter
=
NULL
;
while
(
(
pIter
=
taosHashIterate
(
RSMA_INFO_HASH
(
pRSmaStat
),
pIter
))
)
{
tb_uid_t
*
pSuid
=
(
tb_uid_t
*
)
taosHashGetKey
(
pIter
,
NULL
);
SRSmaInfo
*
pRSmaInfo
=
*
(
SRSmaInfo
**
)
pIter
;
if
(
RSMA_INFO_IS_DEL
(
pRSmaInfo
))
{
...
...
@@ -447,14 +445,13 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
SMA_VID
(
pSma
),
refVal
,
*
pSuid
);
}
pIter
=
taosHashIterate
(
RSMA_INFO_HASH
(
pRSmaStat
),
pIter
);
continue
;
}
#if 0
if (pRSmaInfo->taskInfo[0]) {
if (pRSmaInfo->iTaskInfo[0]) {
SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pRSmaInfo->iTaskInfo[0];
tdFreeRSmaInfo
(
pSma
,
pRSmaInfo
,
tru
e
);
tdFreeRSmaInfo(pSma, pRSmaInfo,
fals
e);
pRSmaInfo->iTaskInfo[0] = NULL;
}
} else {
...
...
@@ -463,8 +460,7 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter));
smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), *pSuid);
pIter
=
taosHashIterate
(
RSMA_INFO_HASH
(
pRSmaStat
),
pIter
);
#endif
}
for
(
int32_t
i
=
0
;
i
<
taosArrayGetSize
(
rsmaDeleted
);
++
i
)
{
...
...
@@ -480,10 +476,9 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
taosHashRemove
(
RSMA_INFO_HASH
(
pRSmaStat
),
pSuid
,
sizeof
(
tb_uid_t
));
}
taosArrayDestroy
(
rsmaDeleted
);
// TODO: remove suid in files?
// unlock
taosWUnLockLatch
(
SMA_ENV_LOCK
(
pEnv
));
//
taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
// step 2: cleanup outdated qtaskinfo files
tdCleanupQTaskInfoFiles
(
pSma
,
pRSmaStat
);
...
...
source/dnode/vnode/src/sma/smaEnv.c
浏览文件 @
4bc8e086
...
...
@@ -209,6 +209,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
SRSmaStat
*
pRSmaStat
=
(
SRSmaStat
*
)(
*
pSmaStat
);
pRSmaStat
->
pSma
=
(
SSma
*
)
pSma
;
atomic_store_8
(
RSMA_TRIGGER_STAT
(
pRSmaStat
),
TASK_TRIGGER_STAT_INIT
);
tsem_init
(
&
pRSmaStat
->
notEmpty
,
0
,
0
);
// init smaMgmt
smaInit
();
...
...
@@ -230,12 +231,6 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
if
(
!
RSMA_INFO_HASH
(
pRSmaStat
))
{
return
TSDB_CODE_FAILED
;
}
RSMA_FETCH_HASH
(
pRSmaStat
)
=
taosHashInit
(
RSMA_TASK_INFO_HASH_SLOT
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BIGINT
),
true
,
HASH_ENTRY_LOCK
);
if
(
!
RSMA_FETCH_HASH
(
pRSmaStat
))
{
return
TSDB_CODE_FAILED
;
}
}
else
if
(
smaType
==
TSDB_SMA_TYPE_TIME_RANGE
)
{
// TODO
}
else
{
...
...
@@ -267,6 +262,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
smaDebug
(
"vgId:%d, destroy rsma stat %p"
,
SMA_VID
(
pSma
),
pRSmaStat
);
// step 1: set rsma trigger stat cancelled
atomic_store_8
(
RSMA_TRIGGER_STAT
(
pStat
),
TASK_TRIGGER_STAT_CANCELLED
);
tsem_destroy
(
&
(
pStat
->
notEmpty
));
// step 2: destroy the rsma info and associated fetch tasks
if
(
taosHashGetSize
(
RSMA_INFO_HASH
(
pStat
))
>
0
)
{
...
...
@@ -279,17 +275,14 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
}
taosHashCleanup
(
RSMA_INFO_HASH
(
pStat
));
// step 3: destroy the rsma fetch hash
taosHashCleanup
(
RSMA_FETCH_HASH
(
pStat
));
// step 4: wait all triggered fetch tasks finished
// step 3: wait for all triggered fetch tasks to finish
int32_t
nLoops
=
0
;
while
(
1
)
{
if
(
T_REF_VAL_GET
((
SSmaStat
*
)
pStat
)
==
0
)
{
smaDebug
(
"vgId:%d, rsma fetch tasks all finished"
,
SMA_VID
(
pSma
));
smaDebug
(
"vgId:%d, rsma fetch tasks a
re a
ll finished"
,
SMA_VID
(
pSma
));
break
;
}
else
{
smaDebug
(
"vgId:%d, rsma fetch tasks not all finished yet"
,
SMA_VID
(
pSma
));
smaDebug
(
"vgId:%d, rsma fetch tasks
are
not all finished yet"
,
SMA_VID
(
pSma
));
}
++
nLoops
;
if
(
nLoops
>
1000
)
{
...
...
source/dnode/vnode/src/sma/smaOpen.c
浏览文件 @
4bc8e086
...
...
@@ -146,6 +146,20 @@ int32_t smaClose(SSma *pSma) {
return
0
;
}
int32_t
smaPreClose
(
SSma
*
pSma
)
{
if
(
pSma
&&
VND_IS_RSMA
(
pSma
->
pVnode
))
{
SSmaEnv
*
pEnv
=
NULL
;
SRSmaStat
*
pStat
=
NULL
;
if
(
!
(
pEnv
=
SMA_RSMA_ENV
(
pSma
))
||
!
(
pStat
=
(
SRSmaStat
*
)
SMA_ENV_STAT
(
pEnv
)))
{
return
0
;
}
for
(
int32_t
i
=
0
;
i
<
RSMA_EXECUTOR_MAX
;
++
i
)
{
tsem_post
(
&
(
pStat
->
notEmpty
));
}
}
return
0
;
}
/**
* @brief rsma env restore
*
...
...
source/dnode/vnode/src/sma/smaRollup.c
浏览文件 @
4bc8e086
此差异已折叠。
点击以展开。
source/dnode/vnode/src/sma/smaUtil.c
浏览文件 @
4bc8e086
...
...
@@ -375,6 +375,9 @@ int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) {
if
(
TABLE_IS_ROLLUP
(
mr
.
me
.
flags
))
{
param
=
&
mr
.
me
.
stbEntry
.
rsmaParam
;
for
(
int32_t
i
=
0
;
i
<
TSDB_RETENTION_L2
;
++
i
)
{
if
(
!
pInfo
->
iTaskInfo
[
i
])
{
continue
;
}
if
(
tdCloneQTaskInfo
(
pSma
,
pInfo
->
taskInfo
[
i
],
pInfo
->
iTaskInfo
[
i
],
param
,
pInfo
->
suid
,
i
)
<
0
)
{
goto
_err
;
}
...
...
source/dnode/vnode/src/vnd/vnodeCommit.c
浏览文件 @
4bc8e086
...
...
@@ -220,13 +220,6 @@ int vnodeCommit(SVnode *pVnode) {
vInfo
(
"vgId:%d, start to commit, commit ID:%"
PRId64
" version:%"
PRId64
,
TD_VID
(
pVnode
),
pVnode
->
state
.
commitID
,
pVnode
->
state
.
applied
);
// preCommit
// smaSyncPreCommit(pVnode->pSma);
smaAsyncPreCommit
(
pVnode
->
pSma
);
vnodeBufPoolUnRef
(
pVnode
->
inUse
);
pVnode
->
inUse
=
NULL
;
pVnode
->
state
.
commitTerm
=
pVnode
->
state
.
applyTerm
;
// save info
...
...
@@ -241,6 +234,16 @@ int vnodeCommit(SVnode *pVnode) {
}
walBeginSnapshot
(
pVnode
->
pWal
,
pVnode
->
state
.
applied
);
// preCommit
// smaSyncPreCommit(pVnode->pSma);
if
(
smaAsyncPreCommit
(
pVnode
->
pSma
)
<
0
){
ASSERT
(
0
);
return
-
1
;
}
vnodeBufPoolUnRef
(
pVnode
->
inUse
);
pVnode
->
inUse
=
NULL
;
// commit each sub-system
if
(
metaCommit
(
pVnode
->
pMeta
)
<
0
)
{
ASSERT
(
0
);
...
...
@@ -248,7 +251,10 @@ int vnodeCommit(SVnode *pVnode) {
}
if
(
VND_IS_RSMA
(
pVnode
))
{
smaAsyncCommit
(
pVnode
->
pSma
);
if
(
smaAsyncCommit
(
pVnode
->
pSma
)
<
0
)
{
ASSERT
(
0
);
return
-
1
;
}
if
(
tsdbCommit
(
VND_RSMA0
(
pVnode
))
<
0
)
{
ASSERT
(
0
);
...
...
@@ -285,7 +291,10 @@ int vnodeCommit(SVnode *pVnode) {
// postCommit
// smaSyncPostCommit(pVnode->pSma);
smaAsyncPostCommit
(
pVnode
->
pSma
);
if
(
smaAsyncPostCommit
(
pVnode
->
pSma
)
<
0
)
{
ASSERT
(
0
);
return
-
1
;
}
// apply the commit (TODO)
walEndSnapshot
(
pVnode
->
pWal
);
...
...
source/dnode/vnode/src/vnd/vnodeOpen.c
浏览文件 @
4bc8e086
...
...
@@ -87,6 +87,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
pVnode
->
msgCb
=
msgCb
;
taosThreadMutexInit
(
&
pVnode
->
lock
,
NULL
);
pVnode
->
blocked
=
false
;
pVnode
->
inClose
=
false
;
tsem_init
(
&
pVnode
->
syncSem
,
0
,
0
);
tsem_init
(
&
(
pVnode
->
canCommit
),
0
,
1
);
...
...
@@ -181,6 +182,8 @@ _err:
void
vnodePreClose
(
SVnode
*
pVnode
)
{
if
(
pVnode
)
{
syncLeaderTransfer
(
pVnode
->
sync
);
pVnode
->
inClose
=
true
;
smaPreClose
(
pVnode
->
pSma
);
}
}
...
...
source/dnode/vnode/src/vnd/vnodeSvr.c
浏览文件 @
4bc8e086
...
...
@@ -301,8 +301,6 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
return
qWorkerProcessQueryMsg
(
&
handle
,
pVnode
->
pQuery
,
pMsg
,
0
);
case
TDMT_SCH_QUERY_CONTINUE
:
return
qWorkerProcessCQueryMsg
(
&
handle
,
pVnode
->
pQuery
,
pMsg
,
0
);
case
TDMT_VND_FETCH_RSMA
:
return
smaProcessFetch
(
pVnode
->
pSma
,
pMsg
);
case
TDMT_VND_EXEC_RSMA
:
return
smaProcessExec
(
pVnode
->
pSma
,
pMsg
);
default:
...
...
source/libs/executor/src/executorimpl.c
浏览文件 @
4bc8e086
...
...
@@ -3137,6 +3137,7 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result) {
initResultRow
(
resultRow
);
pInfo
->
resultRowInfo
.
cur
=
(
SResultRowPosition
){.
pageId
=
resultRow
->
pageId
,
.
offset
=
resultRow
->
offset
};
// releaseBufPage(pSup->pResultBuf, getBufPage(pSup->pResultBuf, pageId));
}
if
(
offset
!=
length
)
{
...
...
source/util/src/tqueue.c
浏览文件 @
4bc8e086
...
...
@@ -298,7 +298,8 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) {
return
num
;
}
void
taosResetQitems
(
STaosQall
*
qall
)
{
qall
->
current
=
qall
->
start
;
}
void
taosResetQitems
(
STaosQall
*
qall
)
{
qall
->
current
=
qall
->
start
;
}
int32_t
taosQallItemSize
(
STaosQall
*
qall
)
{
return
qall
->
numOfItems
;
}
STaosQset
*
taosOpenQset
()
{
STaosQset
*
qset
=
taosMemoryCalloc
(
sizeof
(
STaosQset
),
1
);
...
...
tests/script/tsim/sma/rsmaPersistenceRecovery.sim
浏览文件 @
4bc8e086
...
...
@@ -35,6 +35,7 @@ sleep 7000
print =============== select * from retention level 2 from memory
sql select * from ct1;
print $data00 $data01 $data02
print $data10 $data11 $data12
if $rows > 2 then
print retention level 2 file rows $rows > 2
return -1
...
...
@@ -51,6 +52,7 @@ endi
print =============== select * from retention level 1 from memory
sql select * from ct1 where ts > now-8d;
print $data00 $data01 $data02
print $data10 $data11 $data12
if $rows > 2 then
print retention level 1 file rows $rows > 2
return -1
...
...
@@ -89,6 +91,7 @@ system sh/exec.sh -n dnode1 -s start
print =============== select * from retention level 2 from file
sql select * from ct1;
print $data00 $data01 $data02
print $data10 $data11 $data12
if $rows > 2 then
print retention level 2 file rows $rows > 2
return -1
...
...
@@ -104,6 +107,7 @@ endi
print =============== select * from retention level 1 from file
sql select * from ct1 where ts > now-8d;
print $data00 $data01 $data02
print $data10 $data11 $data12
if $rows > 2 then
print retention level 1 file rows $rows > 2
return -1
...
...
@@ -141,6 +145,7 @@ sleep 7000
print =============== select * from retention level 2 from file and memory after rsma qtaskinfo recovery
sql select * from ct1;
print $data00 $data01 $data02
print $data10 $data11 $data12
if $rows > 2 then
print retention level 2 file/mem rows $rows > 2
return -1
...
...
@@ -163,6 +168,7 @@ endi
print =============== select * from retention level 1 from file and memory after rsma qtaskinfo recovery
sql select * from ct1 where ts > now-8d;
print $data00 $data01 $data02
print $data10 $data11 $data12
if $rows > 2 then
print retention level 1 file/mem rows $rows > 2
return -1
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录