Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
c3b5f435
T
TDengine
项目概览
taosdata
/
TDengine
大约 1 年 前同步成功
通知
1184
Star
22015
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
c3b5f435
编写于
4月 07, 2022
作者:
C
Cary Xu
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
tsma integration with stream computing
上级
d4d9ce20
变更
10
展开全部
隐藏空白更改
内联
并排
Showing
10 changed file
with
241 addition
and
109 deletion
+241
-109
source/dnode/vnode/inc/tsdb.h
source/dnode/vnode/inc/tsdb.h
+9
-5
source/dnode/vnode/src/inc/tsdbDef.h
source/dnode/vnode/src/inc/tsdbDef.h
+12
-10
source/dnode/vnode/src/inc/tsdbSma.h
source/dnode/vnode/src/inc/tsdbSma.h
+10
-2
source/dnode/vnode/src/meta/metaBDBImpl.c
source/dnode/vnode/src/meta/metaBDBImpl.c
+13
-14
source/dnode/vnode/src/tq/tq.c
source/dnode/vnode/src/tq/tq.c
+7
-0
source/dnode/vnode/src/tsdb/tsdbMain.c
source/dnode/vnode/src/tsdb/tsdbMain.c
+5
-5
source/dnode/vnode/src/tsdb/tsdbSma.c
source/dnode/vnode/src/tsdb/tsdbSma.c
+163
-60
source/dnode/vnode/src/vnd/vnodeWrite.c
source/dnode/vnode/src/vnd/vnodeWrite.c
+14
-9
source/dnode/vnode/test/tsdbSmaTest.cpp
source/dnode/vnode/test/tsdbSmaTest.cpp
+1
-1
source/libs/executor/src/executorimpl.c
source/libs/executor/src/executorimpl.c
+7
-3
未找到文件。
source/dnode/vnode/inc/tsdb.h
浏览文件 @
c3b5f435
...
...
@@ -88,14 +88,18 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp);
int
tsdbPrepareCommit
(
STsdb
*
pTsdb
);
int
tsdbCommit
(
STsdb
*
pTsdb
);
int32_t
tsdbInitSma
(
STsdb
*
pTsdb
);
int32_t
tsdbCreateTSma
(
STsdb
*
pTsdb
,
char
*
pMsg
);
int32_t
tsdbDropTSma
(
STsdb
*
pTsdb
,
char
*
pMsg
);
/**
* @brief When submit msg received, update the relative expired window synchronously.
*
* @param pTsdb
* @param msg
* @return int32_t
*
* @param pTsdb
* @param msg
* @return int32_t
*/
int32_t
tsdbUpdateSmaWindow
(
STsdb
*
pTsdb
,
const
char
*
m
sg
);
int32_t
tsdbUpdateSmaWindow
(
STsdb
*
pTsdb
,
SSubmitReq
*
pM
sg
);
/**
* @brief Insert tSma(Time-range-wise SMA) data from stream computing engine
...
...
source/dnode/vnode/src/inc/tsdbDef.h
浏览文件 @
c3b5f435
...
...
@@ -46,7 +46,7 @@ extern "C" {
struct
STsdb
{
int32_t
vgId
;
bool
repoLocked
;
TdThreadMutex
mutex
;
TdThreadMutex
mutex
;
char
*
path
;
STsdbCfg
config
;
STsdbMemTable
*
mem
;
...
...
@@ -56,17 +56,19 @@ struct STsdb {
STsdbFS
*
fs
;
SMeta
*
pMeta
;
STfs
*
pTfs
;
SSmaEnv
*
pTSmaEnv
;
SSmaEnv
*
pRSmaEnv
;
SSmaEnvs
smaEnvs
;
};
#define REPO_ID(r) ((r)->vgId)
#define REPO_CFG(r) (&(r)->config)
#define REPO_FS(r) (r)->fs
#define REPO_META(r) (r)->pMeta
#define REPO_TFS(r) (r)->pTfs
#define IS_REPO_LOCKED(r) (r)->repoLocked
#define REPO_SMA_ENV(r, t) ((TSDB_SMA_TYPE_ROLLUP == (t)) ? (r)->pRSmaEnv : (r)->pTSmaEnv)
#define REPO_ID(r) ((r)->vgId)
#define REPO_CFG(r) (&(r)->config)
#define REPO_FS(r) ((r)->fs)
#define REPO_META(r) ((r)->pMeta)
#define REPO_TFS(r) ((r)->pTfs)
#define IS_REPO_LOCKED(r) ((r)->repoLocked)
#define REPO_TSMA_NUM(r) ((r)->smaEnvs.nTSma)
#define REPO_RSMA_NUM(r) ((r)->smaEnvs.nRSma)
#define REPO_TSMA_ENV(r) ((r)->smaEnvs.pTSmaEnv)
#define REPO_RSMA_ENV(r) ((r)->smaEnvs.pRSmaEnv)
int
tsdbLockRepo
(
STsdb
*
pTsdb
);
int
tsdbUnlockRepo
(
STsdb
*
pTsdb
);
...
...
source/dnode/vnode/src/inc/tsdbSma.h
浏览文件 @
c3b5f435
...
...
@@ -18,8 +18,9 @@
#define TSDB_SMA_TEST // remove after test finished
typedef
struct
SSmaStat
SSmaStat
;
typedef
struct
SSmaEnv
SSmaEnv
;
typedef
struct
SSmaStat
SSmaStat
;
typedef
struct
SSmaEnv
SSmaEnv
;
typedef
struct
SSmaEnvs
SSmaEnvs
;
struct
SSmaEnv
{
TdThreadRwlock
lock
;
...
...
@@ -36,6 +37,13 @@ struct SSmaEnv {
#define SMA_ENV_STAT(env) ((env)->pStat)
#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
struct
SSmaEnvs
{
int16_t
nTSma
;
int16_t
nRSma
;
SSmaEnv
*
pTSmaEnv
;
SSmaEnv
*
pRSmaEnv
;
};
void
tsdbDestroySmaEnv
(
SSmaEnv
*
pSmaEnv
);
void
*
tsdbFreeSmaEnv
(
SSmaEnv
*
pSmaEnv
);
#if 0
...
...
source/dnode/vnode/src/meta/metaBDBImpl.c
浏览文件 @
c3b5f435
...
...
@@ -912,7 +912,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) {
pCur
->
uid
=
uid
;
// TODO: lock?
ret
=
pDB
->
pCtbIdx
->
cursor
(
pDB
->
pSmaIdx
,
NULL
,
&
(
pCur
->
pCur
),
0
);
if
(
ret
!=
0
)
{
if
(
(
ret
!=
0
)
||
(
pCur
->
pCur
==
NULL
)
)
{
taosMemoryFree
(
pCur
);
return
NULL
;
}
...
...
@@ -996,32 +996,31 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
}
SArray
*
metaGetSmaTbUids
(
SMeta
*
pMeta
,
bool
isDup
)
{
SArray
*
pUids
=
NULL
;
SArray
*
pUids
=
NULL
;
SMetaDB
*
pDB
=
pMeta
->
pDB
;
DBC
*
pCur
=
NULL
;
DBC
*
pCur
=
NULL
;
DBT
pkey
=
{
0
},
pval
=
{
0
};
uint32_t
mode
=
isDup
?
DB_NEXT_DUP
:
DB_NEXT_NODUP
;
int
ret
;
pUids
=
taosArrayInit
(
16
,
sizeof
(
tb_uid_t
));
if
(
!
pUids
)
{
return
NULL
;
}
// TODO: lock?
ret
=
pDB
->
pCtbIdx
->
cursor
(
pDB
->
pSmaIdx
,
NULL
,
&
pCur
,
0
);
if
(
ret
!=
0
)
{
taosArrayDestroy
(
pUids
);
return
NULL
;
}
void
*
pBuf
=
NULL
;
// TODO: lock?
while
((
ret
=
pCur
->
get
(
pCur
,
&
pkey
,
&
pval
,
mode
))
==
0
)
{
taosArrayPush
(
pUids
,
pkey
.
data
);
if
(
!
pUids
)
{
pUids
=
taosArrayInit
(
16
,
sizeof
(
tb_uid_t
));
if
(
!
pUids
)
{
return
NULL
;
}
}
taosArrayPush
(
pUids
,
pkey
.
data
);
}
// TODO: lock?
if
(
pCur
)
{
pCur
->
close
(
pCur
);
...
...
source/dnode/vnode/src/tq/tq.c
浏览文件 @
c3b5f435
...
...
@@ -81,6 +81,13 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi
return
-
1
;
}
memcpy
(
data
,
msg
,
msgLen
);
if
(
msgType
==
TDMT_VND_SUBMIT
)
{
if
(
tsdbUpdateSmaWindow
(
pTq
->
pVnode
->
pTsdb
,
msg
)
!=
0
)
{
return
-
1
;
}
}
SRpcMsg
req
=
{
.
msgType
=
TDMT_VND_STREAM_TRIGGER
,
.
pCont
=
data
,
...
...
source/dnode/vnode/src/tsdb/tsdbMain.c
浏览文件 @
c3b5f435
...
...
@@ -80,9 +80,6 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg,
pTsdb
->
pmaf
=
pMAF
;
pTsdb
->
pMeta
=
pMeta
;
pTsdb
->
pTfs
=
pTfs
;
pTsdb
->
pTSmaEnv
=
NULL
;
pTsdb
->
pRSmaEnv
=
NULL
;
pTsdb
->
fs
=
tsdbNewFS
(
pTsdbCfg
);
return
pTsdb
;
...
...
@@ -90,8 +87,8 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg,
static
void
tsdbFree
(
STsdb
*
pTsdb
)
{
if
(
pTsdb
)
{
tsdbFreeSmaEnv
(
pTsdb
->
pRSmaEnv
);
tsdbFreeSmaEnv
(
pTsdb
->
pTSmaEnv
);
tsdbFreeSmaEnv
(
REPO_TSMA_ENV
(
pTsdb
)
);
tsdbFreeSmaEnv
(
REPO_RSMA_ENV
(
pTsdb
)
);
tsdbFreeFS
(
pTsdb
->
fs
);
taosMemoryFreeClear
(
pTsdb
->
path
);
taosMemoryFree
(
pTsdb
);
...
...
@@ -100,7 +97,10 @@ static void tsdbFree(STsdb *pTsdb) {
static
int
tsdbOpenImpl
(
STsdb
*
pTsdb
)
{
tsdbOpenFS
(
pTsdb
);
tsdbInitSma
(
pTsdb
);
// TODO
return
0
;
}
...
...
source/dnode/vnode/src/tsdb/tsdbSma.c
浏览文件 @
c3b5f435
此差异已折叠。
点击以展开。
source/dnode/vnode/src/vnd/vnodeWrite.c
浏览文件 @
c3b5f435
...
...
@@ -202,17 +202,17 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
}
}
break
;
case
TDMT_VND_CREATE_SMA
:
{
// timeRangeSMA
#if
1
#if
0
SSmaCfg vCreateSmaReq = {0};
if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
vWarn
(
"vgId
%d:
TDMT_VND_CREATE_SMA received but deserialize failed since %s"
,
pVnode
->
config
.
vgId
,
vWarn("vgId
:%d
TDMT_VND_CREATE_SMA received but deserialize failed since %s", pVnode->config.vgId,
terrstr(terrno));
return -1;
}
v
Warn
(
"vgId%d: TDMT_VND_CREATE_SMA received for %s:%"
PRIi64
,
pVnode
->
config
.
vgId
,
vCreateSmaReq
.
tSma
.
indexName
,
vCreateSmaReq
.
tSma
.
indexUid
);
v
Debug("vgId:%d TDMT_VND_CREATE_SMA msg received for %s:%" PRIi64, pVnode->config.vgId
,
vCreateSmaReq.tSma.indexName,
vCreateSmaReq.tSma.indexUid);
// record current timezone of server side
vCreateSmaReq.tSma.timezoneInt = tsTimezone;
...
...
@@ -222,19 +222,24 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
tdDestroyTSma(&vCreateSmaReq.tSma);
return -1;
}
// TODO: send msg to stream computing to create tSma
// if ((send msg to stream computing) < 0) {
// tdDestroyTSma(&vCreateSmaReq);
// return -1;
// }
tsdbTSmaAdd(pVnode->pTsdb, 1);
tdDestroyTSma(&vCreateSmaReq.tSma);
// TODO: return directly or go on follow steps?
#endif
if
(
tsdbCreateTSma
(
pVnode
->
pTsdb
,
POINTER_SHIFT
(
pMsg
->
pCont
,
sizeof
(
SMsgHead
)))
<
0
)
{
// TODO
}
}
break
;
case
TDMT_VND_CANCEL_SMA
:
{
// timeRangeSMA
}
break
;
case
TDMT_VND_DROP_SMA
:
{
// timeRangeSMA
if
(
tsdbDropTSma
(
pVnode
->
pTsdb
,
POINTER_SHIFT
(
pMsg
->
pCont
,
sizeof
(
SMsgHead
)))
<
0
)
{
// TODO
}
#if 0
tsdbTSmaSub(pVnode->pTsdb, 1);
SVDropTSmaReq vDropSmaReq = {0};
if (tDeserializeSVDropTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vDropSmaReq) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
...
...
source/dnode/vnode/test/tsdbSmaTest.cpp
浏览文件 @
c3b5f435
...
...
@@ -408,7 +408,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
EXPECT_EQ
(
tdScanAndConvertSubmitMsg
(
pMsg
),
TSDB_CODE_SUCCESS
);
EXPECT_EQ
(
tsdbUpdateSmaWindow
(
pTsdb
,
(
const
char
*
)
pMsg
),
0
);
EXPECT_EQ
(
tsdbUpdateSmaWindow
(
pTsdb
,
pMsg
),
0
);
// init
const
int32_t
tSmaGroupSize
=
4
;
...
...
source/libs/executor/src/executorimpl.c
浏览文件 @
c3b5f435
...
...
@@ -544,8 +544,9 @@ static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultR
existInCurrentResusltRowInfo
=
false
;
// this time window created by other timestamp that does not belongs to current table.
assert
(
pResultRowInfo
->
curPos
==
-
1
);
}
else
if
(
pResultRowInfo
->
size
==
1
)
{
ASSERT
(
0
);
// existInCurrentResusltRowInfo = (pResultRowInfo->pResult[0] == (*p1));
// ASSERT(0);
SResultRowPosition
*
p
=
&
pResultRowInfo
->
pPosition
[
0
];
existInCurrentResusltRowInfo
=
(
p
->
pageId
==
p1
->
pageId
&&
p
->
offset
==
p1
->
offset
);
}
else
{
// check if current pResultRowInfo contains the existInCurrentResusltRowInfo pResultRow
SET_RES_EXT_WINDOW_KEY
(
pSup
->
keyBuf
,
pData
,
bytes
,
tid
,
pResultRowInfo
);
int64_t
*
index
=
taosHashGet
(
pSup
->
pResultRowListSet
,
pSup
->
keyBuf
,
GET_RES_EXT_WINDOW_KEY_LEN
(
bytes
));
...
...
@@ -597,6 +598,8 @@ static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultR
int64_t
index
=
pResultRowInfo
->
curPos
;
SET_RES_EXT_WINDOW_KEY
(
pSup
->
keyBuf
,
pData
,
bytes
,
tid
,
pResultRowInfo
);
taosHashPut
(
pSup
->
pResultRowListSet
,
pSup
->
keyBuf
,
GET_RES_EXT_WINDOW_KEY_LEN
(
bytes
),
&
index
,
POINTER_BYTES
);
}
else
{
pResult
=
getResultRowByPos
(
pResultBuf
,
p1
);
}
// too many time window in query
...
...
@@ -1627,7 +1630,8 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
getNumOfRowsInTimeWindow
(
&
pSDataBlock
->
info
,
tsCols
,
startPos
,
ekey
,
binarySearchForKey
,
NULL
,
TSDB_ORDER_ASC
);
// window start(end) key interpolation
doWindowBorderInterpolation
(
pOperatorInfo
,
pSDataBlock
,
pInfo
->
binfo
.
pCtx
,
pResult
,
&
nextWin
,
startPos
,
forwardStep
,
pInfo
->
order
,
false
);
doWindowBorderInterpolation
(
pOperatorInfo
,
pSDataBlock
,
pInfo
->
binfo
.
pCtx
,
pResult
,
&
nextWin
,
startPos
,
forwardStep
,
pInfo
->
order
,
false
);
updateTimeWindowInfo
(
&
pInfo
->
timeWindowData
,
&
nextWin
,
true
);
doApplyFunctions
(
pInfo
->
binfo
.
pCtx
,
&
nextWin
,
&
pInfo
->
timeWindowData
,
startPos
,
forwardStep
,
tsCols
,
pSDataBlock
->
info
.
rows
,
numOfOutput
,
TSDB_ORDER_ASC
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录