Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
bd5d21b2
T
TDengine
项目概览
taosdata
/
TDengine
大约 1 年 前同步成功
通知
1184
Star
22015
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
bd5d21b2
编写于
5月 18, 2022
作者:
S
Shengliang Guan
浏览文件
操作
浏览文件
下载
差异文件
Merge remote-tracking branch 'origin/3.0' into feature/check
上级
b858688f
6e1d926f
变更
36
展开全部
隐藏空白更改
内联
并排
Showing
36 changed file
with
2353 addition
and
286 deletion
+2353
-286
example/src/tmq.c
example/src/tmq.c
+3
-2
include/client/taos.h
include/client/taos.h
+3
-1
include/common/tmsg.h
include/common/tmsg.h
+13
-0
include/util/taoserror.h
include/util/taoserror.h
+2
-0
source/client/src/tmq.c
source/client/src/tmq.c
+155
-25
source/dnode/mnode/impl/src/mndOffset.c
source/dnode/mnode/impl/src/mndOffset.c
+2
-0
source/dnode/vnode/src/inc/sma.h
source/dnode/vnode/src/inc/sma.h
+1
-1
source/dnode/vnode/src/inc/tsdb.h
source/dnode/vnode/src/inc/tsdb.h
+1
-1
source/dnode/vnode/src/meta/metaEntry.c
source/dnode/vnode/src/meta/metaEntry.c
+4
-4
source/dnode/vnode/src/meta/metaQuery.c
source/dnode/vnode/src/meta/metaQuery.c
+0
-7
source/dnode/vnode/src/sma/smaTDBImpl.c
source/dnode/vnode/src/sma/smaTDBImpl.c
+8
-6
source/dnode/vnode/src/sma/smaTimeRange.c
source/dnode/vnode/src/sma/smaTimeRange.c
+50
-69
source/dnode/vnode/src/tsdb/tsdbOpen.c
source/dnode/vnode/src/tsdb/tsdbOpen.c
+3
-2
source/dnode/vnode/src/tsdb/tsdbRead.c
source/dnode/vnode/src/tsdb/tsdbRead.c
+3
-3
source/dnode/vnode/src/tsdb/tsdbSma.c
source/dnode/vnode/src/tsdb/tsdbSma.c
+5
-5
source/dnode/vnode/src/vnd/vnodeOpen.c
source/dnode/vnode/src/vnd/vnodeOpen.c
+1
-1
source/libs/function/inc/builtinsimpl.h
source/libs/function/inc/builtinsimpl.h
+5
-0
source/libs/function/src/builtins.c
source/libs/function/src/builtins.c
+32
-0
source/libs/function/src/builtinsimpl.c
source/libs/function/src/builtinsimpl.c
+117
-0
source/libs/function/src/tudf.c
source/libs/function/src/tudf.c
+21
-5
source/libs/function/src/udfd.c
source/libs/function/src/udfd.c
+1
-2
source/libs/index/src/indexComm.c
source/libs/index/src/indexComm.c
+72
-47
source/libs/index/src/indexTfile.c
source/libs/index/src/indexTfile.c
+10
-9
source/libs/index/test/jsonUT.cc
source/libs/index/test/jsonUT.cc
+1
-1
source/libs/tdb/src/db/tdbPCache.c
source/libs/tdb/src/db/tdbPCache.c
+38
-25
source/libs/tdb/src/db/tdbPager.c
source/libs/tdb/src/db/tdbPager.c
+1
-1
source/libs/tdb/src/inc/tdbInt.h
source/libs/tdb/src/inc/tdbInt.h
+40
-17
source/util/src/terror.c
source/util/src/terror.c
+3
-0
source/util/src/tlog.c
source/util/src/tlog.c
+2
-1
tests/system-test/2-query/union.py
tests/system-test/2-query/union.py
+39
-45
tests/system-test/2-query/union2.py
tests/system-test/2-query/union2.py
+405
-0
tests/system-test/2-query/union3.py
tests/system-test/2-query/union3.py
+406
-0
tests/system-test/2-query/union4.py
tests/system-test/2-query/union4.py
+406
-0
tests/system-test/7-tmq/subscribeDb1.py
tests/system-test/7-tmq/subscribeDb1.py
+487
-0
tests/system-test/fulltest.sh
tests/system-test/fulltest.sh
+7
-3
tests/test/c/tmqSim.c
tests/test/c/tmqSim.c
+6
-3
未找到文件。
example/src/tmq.c
浏览文件 @
bd5d21b2
...
...
@@ -167,7 +167,7 @@ tmq_t* build_consumer() {
tmq_conf_set
(
conf
,
"td.connect.pass"
,
"taosdata"
);
/*tmq_conf_set(conf, "td.connect.db", "abc1");*/
tmq_conf_set
(
conf
,
"msg.with.table.name"
,
"true"
);
tmq_conf_set_
offset
_commit_cb
(
conf
,
tmq_commit_cb_print
,
NULL
);
tmq_conf_set_
auto
_commit_cb
(
conf
,
tmq_commit_cb_print
,
NULL
);
tmq_t
*
tmq
=
tmq_consumer_new
(
conf
,
NULL
,
0
);
assert
(
tmq
);
return
tmq
;
...
...
@@ -176,6 +176,7 @@ tmq_t* build_consumer() {
tmq_list_t
*
build_topic_list
()
{
tmq_list_t
*
topic_list
=
tmq_list_new
();
tmq_list_append
(
topic_list
,
"topic_ctb_column"
);
/*tmq_list_append(topic_list, "tmq_test_db_multi_insert_topic");*/
return
topic_list
;
}
...
...
@@ -190,7 +191,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
int32_t
cnt
=
0
;
/*clock_t startTime = clock();*/
while
(
running
)
{
TAOS_RES
*
tmqmessage
=
tmq_consumer_poll
(
tmq
,
50
0
);
TAOS_RES
*
tmqmessage
=
tmq_consumer_poll
(
tmq
,
0
);
if
(
tmqmessage
)
{
cnt
++
;
/*printf("get data\n");*/
...
...
include/client/taos.h
浏览文件 @
bd5d21b2
...
...
@@ -233,6 +233,8 @@ DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
DLL_EXPORT
TAOS_RES
*
tmq_consumer_poll
(
tmq_t
*
tmq
,
int64_t
wait_time
);
DLL_EXPORT
tmq_resp_err_t
tmq_consumer_close
(
tmq_t
*
tmq
);
DLL_EXPORT
tmq_resp_err_t
tmq_commit
(
tmq_t
*
tmq
,
const
tmq_topic_vgroup_list_t
*
offsets
,
int32_t
async
);
DLL_EXPORT
void
tmq_commit_async
(
tmq_t
*
tmq
,
const
tmq_topic_vgroup_list_t
*
offsets
,
tmq_commit_cb
*
cb
,
void
*
param
);
DLL_EXPORT
tmq_resp_err_t
tmq_commit_sync
(
tmq_t
*
tmq
,
const
tmq_topic_vgroup_list_t
*
offsets
);
#if 0
DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset);
...
...
@@ -251,7 +253,7 @@ typedef enum tmq_conf_res_t tmq_conf_res_t;
DLL_EXPORT
tmq_conf_t
*
tmq_conf_new
();
DLL_EXPORT
tmq_conf_res_t
tmq_conf_set
(
tmq_conf_t
*
conf
,
const
char
*
key
,
const
char
*
value
);
DLL_EXPORT
void
tmq_conf_destroy
(
tmq_conf_t
*
conf
);
DLL_EXPORT
void
tmq_conf_set_
offset
_commit_cb
(
tmq_conf_t
*
conf
,
tmq_commit_cb
*
cb
,
void
*
param
);
DLL_EXPORT
void
tmq_conf_set_
auto
_commit_cb
(
tmq_conf_t
*
conf
,
tmq_commit_cb
*
cb
,
void
*
param
);
/* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
...
...
include/common/tmsg.h
浏览文件 @
bd5d21b2
...
...
@@ -403,6 +403,19 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWra
return
0
;
}
static
FORCE_INLINE
int32_t
tDecodeSSchemaWrapperEx
(
SDecoder
*
pDecoder
,
SSchemaWrapper
*
pSW
)
{
if
(
tDecodeI32v
(
pDecoder
,
&
pSW
->
nCols
)
<
0
)
return
-
1
;
if
(
tDecodeI32v
(
pDecoder
,
&
pSW
->
sver
)
<
0
)
return
-
1
;
pSW
->
pSchema
=
(
SSchema
*
)
tDecoderMalloc
(
pDecoder
,
pSW
->
nCols
*
sizeof
(
SSchema
));
if
(
pSW
->
pSchema
==
NULL
)
return
-
1
;
for
(
int32_t
i
=
0
;
i
<
pSW
->
nCols
;
i
++
)
{
if
(
tDecodeSSchema
(
pDecoder
,
&
pSW
->
pSchema
[
i
])
<
0
)
return
-
1
;
}
return
0
;
}
STSchema
*
tdGetSTSChemaFromSSChema
(
SSchema
**
pSchema
,
int32_t
nCols
);
typedef
struct
{
...
...
include/util/taoserror.h
浏览文件 @
bd5d21b2
...
...
@@ -668,6 +668,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_UDF_INVALID_STATE TAOS_DEF_ERROR_CODE(0, 0x2906)
#define TSDB_CODE_UDF_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2907)
#define TSDB_CODE_UDF_NO_FUNC_HANDLE TAOS_DEF_ERROR_CODE(0, 0x2908)
#define TSDB_CODE_UDF_INVALID_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x2909)
#define TSDB_CODE_UDF_INVALID_OUTPUT_TYPE TAOS_DEF_ERROR_CODE(0, 0x290A)
#define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000)
#define TSDB_CODE_SML_INVALID_PRECISION_TYPE TAOS_DEF_ERROR_CODE(0, 0x3001)
...
...
source/client/src/tmq.c
浏览文件 @
bd5d21b2
...
...
@@ -182,10 +182,14 @@ typedef struct {
typedef
struct
{
tmq_t
*
tmq
;
int32_t
async
;
int8_t
async
;
int8_t
automatic
;
int8_t
freeOffsets
;
tmq_commit_cb
*
userCb
;
tsem_t
rspSem
;
tmq_resp_err_t
rspErr
;
SArray
*
offsets
;
void
*
userParam
;
}
SMqCommitCbParam
;
tmq_conf_t
*
tmq_conf_new
()
{
...
...
@@ -314,6 +318,142 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
return
sprintf
(
dst
,
"%s:%d"
,
topicName
,
vg
);
}
int32_t
tmqCommitCb
(
void
*
param
,
const
SDataBuf
*
pMsg
,
int32_t
code
)
{
SMqCommitCbParam
*
pParam
=
(
SMqCommitCbParam
*
)
param
;
pParam
->
rspErr
=
code
==
0
?
TMQ_RESP_ERR__SUCCESS
:
TMQ_RESP_ERR__FAIL
;
if
(
pParam
->
async
)
{
if
(
pParam
->
automatic
&&
pParam
->
tmq
->
commitCb
)
{
pParam
->
tmq
->
commitCb
(
pParam
->
tmq
,
pParam
->
rspErr
,
(
tmq_topic_vgroup_list_t
*
)
pParam
->
offsets
,
pParam
->
tmq
->
commitCbUserParam
);
}
else
if
(
!
pParam
->
automatic
&&
pParam
->
userCb
)
{
pParam
->
userCb
(
pParam
->
tmq
,
pParam
->
rspErr
,
(
tmq_topic_vgroup_list_t
*
)
pParam
->
offsets
,
pParam
->
userParam
);
}
if
(
pParam
->
freeOffsets
)
{
taosArrayDestroy
(
pParam
->
offsets
);
}
taosMemoryFree
(
pParam
);
}
else
{
tsem_post
(
&
pParam
->
rspSem
);
}
return
0
;
}
int32_t
tmqCommitInner
(
tmq_t
*
tmq
,
const
tmq_topic_vgroup_list_t
*
offsets
,
int8_t
automatic
,
int8_t
async
,
tmq_commit_cb
*
userCb
,
void
*
userParam
)
{
SMqCMCommitOffsetReq
req
;
SArray
*
pOffsets
=
NULL
;
void
*
buf
=
NULL
;
SMqCommitCbParam
*
pParam
=
NULL
;
SMsgSendInfo
*
sendInfo
=
NULL
;
int8_t
freeOffsets
;
int32_t
code
=
-
1
;
if
(
offsets
==
NULL
)
{
freeOffsets
=
1
;
pOffsets
=
taosArrayInit
(
0
,
sizeof
(
SMqOffset
));
for
(
int32_t
i
=
0
;
i
<
taosArrayGetSize
(
tmq
->
clientTopics
);
i
++
)
{
SMqClientTopic
*
pTopic
=
taosArrayGet
(
tmq
->
clientTopics
,
i
);
for
(
int32_t
j
=
0
;
j
<
taosArrayGetSize
(
pTopic
->
vgs
);
j
++
)
{
SMqClientVg
*
pVg
=
taosArrayGet
(
pTopic
->
vgs
,
j
);
SMqOffset
offset
;
tstrncpy
(
offset
.
topicName
,
pTopic
->
topicName
,
TSDB_TOPIC_FNAME_LEN
);
tstrncpy
(
offset
.
cgroup
,
tmq
->
groupId
,
TSDB_CGROUP_LEN
);
offset
.
vgId
=
pVg
->
vgId
;
offset
.
offset
=
pVg
->
currentOffset
;
taosArrayPush
(
pOffsets
,
&
offset
);
}
}
}
else
{
freeOffsets
=
0
;
pOffsets
=
(
SArray
*
)
&
offsets
->
container
;
}
req
.
num
=
(
int32_t
)
pOffsets
->
size
;
req
.
offsets
=
pOffsets
->
pData
;
SEncoder
encoder
;
tEncoderInit
(
&
encoder
,
NULL
,
0
);
code
=
tEncodeSMqCMCommitOffsetReq
(
&
encoder
,
&
req
);
if
(
code
<
0
)
{
goto
END
;
}
int32_t
tlen
=
encoder
.
pos
;
buf
=
taosMemoryMalloc
(
tlen
);
if
(
buf
==
NULL
)
{
tEncoderClear
(
&
encoder
);
goto
END
;
}
tEncoderClear
(
&
encoder
);
tEncoderInit
(
&
encoder
,
buf
,
tlen
);
tEncodeSMqCMCommitOffsetReq
(
&
encoder
,
&
req
);
tEncoderClear
(
&
encoder
);
pParam
=
taosMemoryCalloc
(
1
,
sizeof
(
SMqCommitCbParam
));
if
(
pParam
==
NULL
)
{
goto
END
;
}
pParam
->
tmq
=
tmq
;
pParam
->
automatic
=
automatic
;
pParam
->
async
=
async
;
pParam
->
offsets
=
pOffsets
;
pParam
->
freeOffsets
=
freeOffsets
;
pParam
->
userCb
=
userCb
;
pParam
->
userParam
=
userParam
;
if
(
!
async
)
tsem_init
(
&
pParam
->
rspSem
,
0
,
0
);
sendInfo
=
taosMemoryMalloc
(
sizeof
(
SMsgSendInfo
));
if
(
sendInfo
==
NULL
)
goto
END
;
sendInfo
->
msgInfo
=
(
SDataBuf
){
.
pData
=
buf
,
.
len
=
tlen
,
.
handle
=
NULL
,
};
sendInfo
->
requestId
=
generateRequestId
();
sendInfo
->
requestObjRefId
=
0
;
sendInfo
->
param
=
pParam
;
sendInfo
->
fp
=
tmqCommitCb
;
sendInfo
->
msgType
=
TDMT_MND_MQ_COMMIT_OFFSET
;
SEpSet
epSet
=
getEpSet_s
(
&
tmq
->
pTscObj
->
pAppInfo
->
mgmtEp
);
int64_t
transporterId
=
0
;
asyncSendMsgToServer
(
tmq
->
pTscObj
->
pAppInfo
->
pTransporter
,
&
epSet
,
&
transporterId
,
sendInfo
);
if
(
!
async
)
{
tsem_wait
(
&
pParam
->
rspSem
);
code
=
pParam
->
rspErr
;
tsem_destroy
(
&
pParam
->
rspSem
);
taosMemoryFree
(
pParam
);
}
// avoid double free if msg is sent
buf
=
NULL
;
code
=
0
;
END:
if
(
buf
)
taosMemoryFree
(
buf
);
/*if (pParam) taosMemoryFree(pParam);*/
/*if (sendInfo) taosMemoryFree(sendInfo);*/
if
(
code
!=
0
&&
async
)
{
if
(
automatic
)
{
tmq
->
commitCb
(
tmq
,
TMQ_RESP_ERR__FAIL
,
(
tmq_topic_vgroup_list_t
*
)
pOffsets
,
tmq
->
commitCbUserParam
);
}
else
{
userCb
(
tmq
,
TMQ_RESP_ERR__FAIL
,
(
tmq_topic_vgroup_list_t
*
)
pOffsets
,
userParam
);
}
}
if
(
!
async
&&
freeOffsets
)
{
taosArrayDestroy
(
pOffsets
);
}
return
code
;
}
void
tmqAssignDelayedHbTask
(
void
*
param
,
void
*
tmrId
)
{
tmq_t
*
tmq
=
(
tmq_t
*
)
param
;
int8_t
*
pTaskType
=
taosAllocateQitem
(
sizeof
(
int8_t
),
DEF_QITEM
);
...
...
@@ -350,7 +490,8 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) {
tmqAskEp
(
tmq
,
true
);
taosTmrReset
(
tmqAssignDelayedHbTask
,
1000
,
tmq
,
tmqMgmt
.
timer
,
&
tmq
->
hbTimer
);
}
else
if
(
*
pTaskType
==
TMQ_DELAYED_TASK__COMMIT
)
{
tmq_commit
(
tmq
,
NULL
,
true
);
/*tmq_commit(tmq, NULL, true);*/
tmqCommitInner
(
tmq
,
NULL
,
1
,
1
,
tmq
->
commitCb
,
tmq
->
commitCbUserParam
);
taosTmrReset
(
tmqAssignDelayedCommitTask
,
tmq
->
autoCommitInterval
,
tmq
,
tmqMgmt
.
timer
,
&
tmq
->
commitTimer
);
}
else
if
(
*
pTaskType
==
TMQ_DELAYED_TASK__REPORT
)
{
}
else
{
...
...
@@ -385,32 +526,11 @@ void tmqClearUnhandleMsg(tmq_t* tmq) {
int32_t
tmqSubscribeCb
(
void
*
param
,
const
SDataBuf
*
pMsg
,
int32_t
code
)
{
SMqSubscribeCbParam
*
pParam
=
(
SMqSubscribeCbParam
*
)
param
;
pParam
->
rspErr
=
code
;
tmq_t
*
tmq
=
pParam
->
tmq
;
/*tmq_t* tmq = pParam->tmq;*/
tsem_post
(
&
pParam
->
rspSem
);
return
0
;
}
int32_t
tmqCommitCb
(
void
*
param
,
const
SDataBuf
*
pMsg
,
int32_t
code
)
{
SMqCommitCbParam
*
pParam
=
(
SMqCommitCbParam
*
)
param
;
pParam
->
rspErr
=
code
==
0
?
TMQ_RESP_ERR__SUCCESS
:
TMQ_RESP_ERR__FAIL
;
if
(
pParam
->
tmq
->
commitCb
)
{
pParam
->
tmq
->
commitCb
(
pParam
->
tmq
,
pParam
->
rspErr
,
NULL
,
pParam
->
tmq
->
commitCbUserParam
);
}
if
(
!
pParam
->
async
)
tsem_post
(
&
pParam
->
rspSem
);
else
{
if
(
pParam
->
offsets
)
{
taosArrayDestroy
(
pParam
->
offsets
);
}
tsem_destroy
(
&
pParam
->
rspSem
);
/*if (pParam->pArray) {*/
/*taosArrayDestroy(pParam->pArray);*/
/*}*/
taosMemoryFree
(
pParam
);
}
return
0
;
}
tmq_resp_err_t
tmq_subscription
(
tmq_t
*
tmq
,
tmq_list_t
**
topics
)
{
if
(
*
topics
==
NULL
)
{
*
topics
=
tmq_list_new
();
...
...
@@ -541,6 +661,8 @@ FAIL:
}
tmq_resp_err_t
tmq_commit
(
tmq_t
*
tmq
,
const
tmq_topic_vgroup_list_t
*
offsets
,
int32_t
async
)
{
return
tmqCommitInner
(
tmq
,
offsets
,
0
,
async
,
tmq
->
commitCb
,
tmq
->
commitCbUserParam
);
#if 0
// TODO: add read write lock
SRequestObj* pRequest = NULL;
tmq_resp_err_t resp = TMQ_RESP_ERR__SUCCESS;
...
...
@@ -627,6 +749,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in
}
return resp;
#endif
}
tmq_resp_err_t
tmq_subscribe
(
tmq_t
*
tmq
,
const
tmq_list_t
*
topic_list
)
{
...
...
@@ -723,7 +846,7 @@ FAIL:
return
code
;
}
void
tmq_conf_set_
offset
_commit_cb
(
tmq_conf_t
*
conf
,
tmq_commit_cb
*
cb
,
void
*
param
)
{
void
tmq_conf_set_
auto
_commit_cb
(
tmq_conf_t
*
conf
,
tmq_commit_cb
*
cb
,
void
*
param
)
{
//
conf
->
commitCb
=
cb
;
conf
->
commitCbUserParam
=
param
;
...
...
@@ -1384,3 +1507,10 @@ const char* tmq_get_table_name(TAOS_RES* res) {
}
return
NULL
;
}
DLL_EXPORT
void
tmq_commit_async
(
tmq_t
*
tmq
,
const
tmq_topic_vgroup_list_t
*
offsets
,
tmq_commit_cb
*
cb
,
void
*
param
)
{
tmqCommitInner
(
tmq
,
offsets
,
0
,
1
,
cb
,
param
);
}
DLL_EXPORT
tmq_resp_err_t
tmq_commit_sync
(
tmq_t
*
tmq
,
const
tmq_topic_vgroup_list_t
*
offsets
)
{
return
tmqCommitInner
(
tmq
,
offsets
,
0
,
0
,
NULL
,
NULL
);
}
source/dnode/mnode/impl/src/mndOffset.c
浏览文件 @
bd5d21b2
...
...
@@ -196,6 +196,8 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) {
}
}
tDecoderClear
(
&
decoder
);
if
(
mndTransPrepare
(
pMnode
,
pTrans
)
!=
0
)
{
mError
(
"mq-commit-offset-trans:%d, failed to prepare since %s"
,
pTrans
->
id
,
terrstr
());
mndTransDrop
(
pTrans
);
...
...
source/dnode/vnode/src/inc/sma.h
浏览文件 @
bd5d21b2
...
...
@@ -70,7 +70,7 @@ struct SSmaStatItem {
* N.B. only applicable to tsma
*/
int8_t
state
;
// ETsdbSmaStat
SHashObj
*
expiredWindows
;
// key: skey of time window, value:
N/A
SHashObj
*
expiredWindows
;
// key: skey of time window, value:
version
STSma
*
pTSma
;
// cache schema
};
...
...
source/dnode/vnode/src/inc/tsdb.h
浏览文件 @
bd5d21b2
...
...
@@ -60,7 +60,7 @@ typedef struct {
TSKEY
minKey
;
}
SRtn
;
#define TSDB_DATA_DIR_LEN 6
#define TSDB_DATA_DIR_LEN 6
// adapt accordingly
struct
STsdb
{
char
*
path
;
SVnode
*
pVnode
;
...
...
source/dnode/vnode/src/meta/metaEntry.c
浏览文件 @
bd5d21b2
...
...
@@ -56,8 +56,8 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
if
(
tDecodeCStr
(
pCoder
,
&
pME
->
name
)
<
0
)
return
-
1
;
if
(
pME
->
type
==
TSDB_SUPER_TABLE
)
{
if
(
tDecodeSSchemaWrapper
(
pCoder
,
&
pME
->
stbEntry
.
schema
)
<
0
)
return
-
1
;
if
(
tDecodeSSchemaWrapper
(
pCoder
,
&
pME
->
stbEntry
.
schemaTag
)
<
0
)
return
-
1
;
if
(
tDecodeSSchemaWrapper
Ex
(
pCoder
,
&
pME
->
stbEntry
.
schema
)
<
0
)
return
-
1
;
if
(
tDecodeSSchemaWrapper
Ex
(
pCoder
,
&
pME
->
stbEntry
.
schemaTag
)
<
0
)
return
-
1
;
}
else
if
(
pME
->
type
==
TSDB_CHILD_TABLE
)
{
if
(
tDecodeI64
(
pCoder
,
&
pME
->
ctbEntry
.
ctime
)
<
0
)
return
-
1
;
if
(
tDecodeI32
(
pCoder
,
&
pME
->
ctbEntry
.
ttlDays
)
<
0
)
return
-
1
;
...
...
@@ -67,9 +67,9 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
if
(
tDecodeI64
(
pCoder
,
&
pME
->
ntbEntry
.
ctime
)
<
0
)
return
-
1
;
if
(
tDecodeI32
(
pCoder
,
&
pME
->
ntbEntry
.
ttlDays
)
<
0
)
return
-
1
;
if
(
tDecodeI32v
(
pCoder
,
&
pME
->
ntbEntry
.
ncid
)
<
0
)
return
-
1
;
if
(
tDecodeSSchemaWrapper
(
pCoder
,
&
pME
->
ntbEntry
.
schema
)
<
0
)
return
-
1
;
if
(
tDecodeSSchemaWrapper
Ex
(
pCoder
,
&
pME
->
ntbEntry
.
schema
)
<
0
)
return
-
1
;
}
else
if
(
pME
->
type
==
TSDB_TSMA_TABLE
)
{
pME
->
smaEntry
.
tsma
=
t
aosMemoryCalloc
(
1
,
sizeof
(
STSma
));
pME
->
smaEntry
.
tsma
=
t
DecoderMalloc
(
pCoder
,
sizeof
(
STSma
));
if
(
!
pME
->
smaEntry
.
tsma
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
return
-
1
;
...
...
source/dnode/vnode/src/meta/metaQuery.c
浏览文件 @
bd5d21b2
...
...
@@ -394,11 +394,6 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) {
goto
_err
;
}
SMSmaCursor
*
pCur
=
metaOpenSmaCursor
(
pMeta
,
uid
);
if
(
pCur
==
NULL
)
{
goto
_err
;
}
SMetaReader
mr
=
{
0
};
metaReaderInit
(
&
mr
,
pMeta
,
0
);
int64_t
smaId
;
...
...
@@ -442,12 +437,10 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) {
metaReaderClear
(
&
mr
);
taosArrayDestroy
(
pSmaIds
);
metaCloseSmaCursor
(
pCur
);
return
pSW
;
_err:
metaReaderClear
(
&
mr
);
taosArrayDestroy
(
pSmaIds
);
metaCloseSmaCursor
(
pCur
);
tdFreeTSmaWrapper
(
pSW
,
deepCopy
);
return
NULL
;
}
...
...
source/dnode/vnode/src/sma/smaTDBImpl.c
浏览文件 @
bd5d21b2
...
...
@@ -55,12 +55,13 @@ static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int
}
static
int32_t
smaOpenDBDb
(
TDB
**
ppDB
,
TENV
*
pEnv
,
const
char
*
pFName
)
{
int
ret
;
tdb_cmpr_fn_t
compFunc
;
// Create a database
compFunc
=
tdSmaKeyCmpr
;
ret
=
tdbDbOpen
(
pFName
,
-
1
,
-
1
,
compFunc
,
pEnv
,
ppDB
);
if
(
tdbDbOpen
(
pFName
,
-
1
,
-
1
,
compFunc
,
pEnv
,
ppDB
)
<
0
)
{
return
-
1
;
}
return
0
;
}
...
...
@@ -76,7 +77,7 @@ int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF) {
// Open DBF
if
(
smaOpenDBDb
(
&
(
pDBF
->
pDB
),
pEnv
,
pDBF
->
path
)
<
0
)
{
terrno
=
TSDB_CODE_TDB_INIT_FAILED
;
smaError
(
"failed to open DBF: %s"
,
pDBF
->
path
)
;
smaCloseDBDb
(
pDBF
->
pDB
);
return
-
1
;
}
...
...
@@ -97,9 +98,10 @@ int32_t smaCloseDBF(SDBFile *pDBF) {
int32_t
smaSaveSmaToDB
(
SDBFile
*
pDBF
,
void
*
pKey
,
int32_t
keyLen
,
void
*
pVal
,
int32_t
valLen
,
TXN
*
txn
)
{
int32_t
ret
;
ret
=
tdbDbInsert
(
pDBF
->
pDB
,
pKey
,
keyLen
,
pVal
,
valLen
,
txn
);
printf
(
"save tsma data into %s, keyLen:%d valLen:%d txn:%p
\n
"
,
pDBF
->
path
,
keyLen
,
valLen
,
txn
);
ret
=
tdbDbUpsert
(
pDBF
->
pDB
,
pKey
,
keyLen
,
pVal
,
valLen
,
txn
);
if
(
ret
<
0
)
{
smaError
(
"failed to
create insert
sma data into db, ret = %d"
,
ret
);
smaError
(
"failed to
upsert t
sma data into db, ret = %d"
,
ret
);
return
-
1
;
}
...
...
@@ -113,7 +115,7 @@ void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_
ret
=
tdbDbGet
(
pDBF
->
pDB
,
pKey
,
keyLen
,
&
pVal
,
valLen
);
if
(
ret
<
0
)
{
smaError
(
"failed to get sma data from db, ret = %d"
,
ret
);
smaError
(
"failed to get
t
sma data from db, ret = %d"
,
ret
);
return
NULL
;
}
...
...
source/dnode/vnode/src/sma/smaTimeRange.c
浏览文件 @
bd5d21b2
...
...
@@ -16,21 +16,22 @@
#include "sma.h"
#include "tsdb.h"
typedef
STsdbCfg
STSmaKeepCfg
;
#undef _TEST_SMA_PRINT_DEBUG_LOG_
#define SMA_STORAGE_TSDB_
DAYS 3
0
#define SMA_STORAGE_TSDB_TIMES 10
#define SMA_STORAGE_SPLIT_
HOURS 24
#define SMA_KEY_LEN
16
// TSKEY+groupId 8+8
#define SMA_DROP_EXPIRED_TIME
10
// default is 10 seconds
#define SMA_STORAGE_TSDB_
MINUTES 8640
0
#define SMA_STORAGE_TSDB_TIMES
10
#define SMA_STORAGE_SPLIT_
FACTOR 144 // least records in tsma file
#define SMA_KEY_LEN
16
// TSKEY+groupId 8+8
#define SMA_DROP_EXPIRED_TIME
10
// default is 10 seconds
#define SMA_STATE_ITEM_HASH_SLOT 32
typedef
struct
{
SSma
*
pSma
;
SDBFile
dFile
;
const
SArray
*
pDataBlocks
;
// sma data
int
32
_t
interval
;
// interval with the precision of DB
int
64
_t
interval
;
// interval with the precision of DB
}
STSmaWriteH
;
typedef
struct
{
...
...
@@ -42,10 +43,10 @@ typedef struct {
STsdb
*
pTsdb
;
SSma
*
pSma
;
SDBFile
dFile
;
int
32
_t
interval
;
// interval with the precision of DB
int
64
_t
interval
;
// interval with the precision of DB
int32_t
blockSize
;
// size of SMA block item
int32_t
days
;
int8_t
storageLevel
;
int8_t
days
;
SmaFsIter
smaFsIter
;
}
STSmaReadH
;
...
...
@@ -58,9 +59,9 @@ typedef enum {
// static func
static
int64_t
tdGetIntervalByPrecision
(
int64_t
interval
,
uint8_t
intervalUnit
,
int8_t
precision
,
bool
adjusted
);
static
int32_t
tdGetSmaStorageLevel
(
int64_t
interval
,
int8_t
intervalUnit
);
static
int32_t
tdGetSmaStorageLevel
(
STSmaKeepCfg
*
pCfg
,
int64_t
interval
);
static
int32_t
tdInitTSmaWriteH
(
STSmaWriteH
*
pSmaH
,
SSma
*
pSma
,
const
SArray
*
pDataBlocks
,
int64_t
interval
,
int8_t
intervalUnit
);
int8_t
intervalUnit
);
static
int32_t
tdInitTSmaReadH
(
STSmaReadH
*
pSmaH
,
SSma
*
pSma
,
int64_t
interval
,
int8_t
intervalUnit
);
static
void
tdDestroyTSmaWriteH
(
STSmaWriteH
*
pSmaH
);
static
int32_t
tdGetTSmaDays
(
SSma
*
pSma
,
int64_t
interval
,
int32_t
storageLevel
);
...
...
@@ -92,9 +93,10 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid);
* @return int32_t
*/
static
int32_t
tdInitTSmaReadH
(
STSmaReadH
*
pSmaH
,
SSma
*
pSma
,
int64_t
interval
,
int8_t
intervalUnit
)
{
STSmaKeepCfg
*
pCfg
=
SMA_TSDB_CFG
(
pSma
);
pSmaH
->
pSma
=
pSma
;
pSmaH
->
interval
=
tdGetIntervalByPrecision
(
interval
,
intervalUnit
,
SMA_TSDB_CFG
(
pSma
)
->
precision
,
true
);
pSmaH
->
storageLevel
=
tdGetSmaStorageLevel
(
interval
,
intervalUnit
);
pSmaH
->
storageLevel
=
tdGetSmaStorageLevel
(
pCfg
,
interval
);
pSmaH
->
days
=
tdGetTSmaDays
(
pSma
,
pSmaH
->
interval
,
pSmaH
->
storageLevel
);
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -275,11 +277,13 @@ static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t f
*/
static
int32_t
tdGetTSmaDays
(
SSma
*
pSma
,
int64_t
interval
,
int32_t
storageLevel
)
{
STsdbCfg
*
pCfg
=
SMA_TSDB_CFG
(
pSma
);
int32_t
daysPerFile
=
pCfg
->
days
;
int32_t
daysPerFile
=
pCfg
->
days
;
// unit is minute
if
(
storageLevel
==
SMA_STORAGE_LEVEL_TSDB
)
{
int32_t
days
=
SMA_STORAGE_TSDB_TIMES
*
(
interval
/
tsTickPerMin
[
pCfg
->
precision
]);
daysPerFile
=
days
>
SMA_STORAGE_TSDB_DAYS
?
days
:
SMA_STORAGE_TSDB_DAYS
;
int32_t
minutes
=
SMA_STORAGE_TSDB_TIMES
*
(
interval
/
tsTickPerMin
[
pCfg
->
precision
]);
if
(
minutes
>
SMA_STORAGE_TSDB_MINUTES
)
{
daysPerFile
=
SMA_STORAGE_TSDB_MINUTES
;
}
}
return
daysPerFile
;
...
...
@@ -288,45 +292,14 @@ static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel)
/**
* @brief Judge the tSma storage level
*
* @param pCfg
* @param interval
* @param intervalUnit
* @return int32_t
*/
static
int32_t
tdGetSmaStorageLevel
(
int64_t
interval
,
int8_t
intervalUnit
)
{
// TODO: configurable for SMA_STORAGE_SPLIT_HOURS?
switch
(
intervalUnit
)
{
case
TIME_UNIT_HOUR
:
if
(
interval
<
SMA_STORAGE_SPLIT_HOURS
)
{
return
SMA_STORAGE_LEVEL_DFILESET
;
}
break
;
case
TIME_UNIT_MINUTE
:
if
(
interval
<
60
*
SMA_STORAGE_SPLIT_HOURS
)
{
return
SMA_STORAGE_LEVEL_DFILESET
;
}
break
;
case
TIME_UNIT_SECOND
:
if
(
interval
<
3600
*
SMA_STORAGE_SPLIT_HOURS
)
{
return
SMA_STORAGE_LEVEL_DFILESET
;
}
break
;
case
TIME_UNIT_MILLISECOND
:
if
(
interval
<
3600
*
1e3
*
SMA_STORAGE_SPLIT_HOURS
)
{
return
SMA_STORAGE_LEVEL_DFILESET
;
}
break
;
case
TIME_UNIT_MICROSECOND
:
if
(
interval
<
3600
*
1e6
*
SMA_STORAGE_SPLIT_HOURS
)
{
return
SMA_STORAGE_LEVEL_DFILESET
;
}
break
;
case
TIME_UNIT_NANOSECOND
:
if
(
interval
<
3600
*
1e9
*
SMA_STORAGE_SPLIT_HOURS
)
{
return
SMA_STORAGE_LEVEL_DFILESET
;
}
break
;
default:
break
;
static
int32_t
tdGetSmaStorageLevel
(
STSmaKeepCfg
*
pCfg
,
int64_t
interval
)
{
int64_t
mInterval
=
convertTimeFromPrecisionToUnit
(
interval
,
pCfg
->
precision
,
TIME_UNIT_MINUTE
);
if
(
pCfg
->
days
/
mInterval
>=
SMA_STORAGE_SPLIT_FACTOR
)
{
return
SMA_STORAGE_LEVEL_DFILESET
;
}
return
SMA_STORAGE_LEVEL_TSDB
;
}
...
...
@@ -346,6 +319,7 @@ static int32_t tdGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
int32_t
tdProcessTSmaInsertImpl
(
SSma
*
pSma
,
int64_t
indexUid
,
const
char
*
msg
)
{
STsdbCfg
*
pCfg
=
SMA_TSDB_CFG
(
pSma
);
const
SArray
*
pDataBlocks
=
(
const
SArray
*
)
msg
;
int64_t
testSkey
=
TSKEY_INITIAL_VAL
;
// TODO: destroy SSDataBlocks(msg)
...
...
@@ -403,8 +377,8 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
}
// Step 1: Judge the storage level and days
int32_t
storageLevel
=
tdGetSmaStorageLevel
(
p
TSma
->
interval
,
pTSma
->
intervalUnit
);
int32_t
days
PerFile
=
tdGetTSmaDays
(
pSma
,
tSmaH
.
interval
,
storageLevel
);
int32_t
storageLevel
=
tdGetSmaStorageLevel
(
p
Cfg
,
tSmaH
.
interval
);
int32_t
minute
PerFile
=
tdGetTSmaDays
(
pSma
,
tSmaH
.
interval
,
storageLevel
);
char
smaKey
[
SMA_KEY_LEN
]
=
{
0
};
// key: skey + groupId
char
dataBuf
[
512
]
=
{
0
};
// val: aggr data // TODO: handle 512 buffer?
...
...
@@ -432,6 +406,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
if
(
!
isStartKey
)
{
isStartKey
=
true
;
skey
=
*
(
TSKEY
*
)
var
;
testSkey
=
skey
;
printf
(
"= skey %"
PRIi64
" groupId = %"
PRIi64
"|"
,
skey
,
groupId
);
tdEncodeTSmaKey
(
groupId
,
skey
,
&
pSmaKey
);
}
else
{
...
...
@@ -503,9 +478,10 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
break
;
}
}
printf
(
"
\n
"
);
// if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) {
if
(
tlen
>
0
)
{
int32_t
fid
=
(
int32_t
)(
TSDB_KEY_FID
(
skey
,
days
PerFile
,
pCfg
->
precision
));
int32_t
fid
=
(
int32_t
)(
TSDB_KEY_FID
(
skey
,
minute
PerFile
,
pCfg
->
precision
));
// Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index
// file
...
...
@@ -517,6 +493,8 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
smaCloseDBF
(
&
tSmaH
.
dFile
);
}
tdSetTSmaDataFile
(
&
tSmaH
,
indexUid
,
fid
);
smaDebug
(
"@@@ vgId:%d write to DBF %s, days:%d, interval:%"
PRIi64
", storageLevel:%"
PRIi32
" queryKey:%"
PRIi64
,
SMA_VID
(
pSma
),
tSmaH
.
dFile
.
path
,
minutePerFile
,
tSmaH
.
interval
,
storageLevel
,
testSkey
);
if
(
smaOpenDBF
(
pEnv
->
dbEnv
,
&
tSmaH
.
dFile
)
!=
0
)
{
smaWarn
(
"vgId:%d open DB file %s failed since %s"
,
SMA_VID
(
pSma
),
tSmaH
.
dFile
.
path
?
tSmaH
.
dFile
.
path
:
"path is NULL"
,
tstrerror
(
terrno
));
...
...
@@ -528,16 +506,17 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
}
if
(
tdInsertTSmaBlocks
(
&
tSmaH
,
&
smaKey
,
SMA_KEY_LEN
,
dataBuf
,
tlen
,
&
pEnv
->
txn
)
!=
0
)
{
smaWarn
(
"vgId:%d insert t
S
ma data blocks fail for index %"
PRIi64
", skey %"
PRIi64
", groupId %"
PRIi64
" since %s"
,
SMA_VID
(
pSma
),
indexUid
,
skey
,
groupId
,
tstrerror
(
terrno
));
smaWarn
(
"vgId:%d insert t
s
ma data blocks fail for index %"
PRIi64
", skey %"
PRIi64
", groupId %"
PRIi64
" since %s"
,
SMA_VID
(
pSma
),
indexUid
,
skey
,
groupId
,
tstrerror
(
terrno
));
tdSmaEndCommit
(
pEnv
);
tdDestroyTSmaWriteH
(
&
tSmaH
);
tdUnRefSmaStat
(
pSma
,
pStat
);
return
TSDB_CODE_FAILED
;
}
smaDebug
(
"vgId:%d insert tSma data blocks success for index %"
PRIi64
", skey %"
PRIi64
", groupId %"
PRIi64
,
SMA_VID
(
pSma
),
indexUid
,
skey
,
groupId
);
smaDebug
(
"vgId:%d insert tsma data blocks success for index %"
PRIi64
", skey %"
PRIi64
", groupId %"
PRIi64
,
SMA_VID
(
pSma
),
indexUid
,
skey
,
groupId
);
// TODO:tsdbEndTSmaCommit();
// Step 3: reset the SSmaStat
...
...
@@ -547,7 +526,6 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
SMA_VID
(
pSma
),
skey
,
tlen
,
indexUid
);
}
printf
(
"
\n
"
);
}
}
tdSmaEndCommit
(
pEnv
);
// TODO: not commit for every insert
...
...
@@ -579,14 +557,14 @@ static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyL
TXN
*
txn
)
{
SDBFile
*
pDBFile
=
&
pSmaH
->
dFile
;
// TODO: insert sma data blocks into B+Tree(TDB)
// TODO: insert
t
sma data blocks into B+Tree(TDB)
if
(
smaSaveSmaToDB
(
pDBFile
,
smaKey
,
keyLen
,
pData
,
dataLen
,
txn
)
!=
0
)
{
smaWarn
(
"vgId:%d insert sma data blocks into %s: smaKey %"
PRIx64
"-%"
PRIx64
", dataLen %"
PRIu32
" fail"
,
SMA_VID
(
pSmaH
->
pSma
),
pDBFile
->
path
,
*
(
int64_t
*
)
smaKey
,
*
(
int64_t
*
)
POINTER_SHIFT
(
smaKey
,
8
),
dataLen
);
smaWarn
(
"vgId:%d insert
t
sma data blocks into %s: smaKey %"
PRIx64
"-%"
PRIx64
", dataLen %"
PRIu32
" fail"
,
SMA_VID
(
pSmaH
->
pSma
),
pDBFile
->
path
,
*
(
int64_t
*
)
smaKey
,
*
(
int64_t
*
)
POINTER_SHIFT
(
smaKey
,
8
),
dataLen
);
return
TSDB_CODE_FAILED
;
}
smaDebug
(
"vgId:%d insert sma data blocks into %s: smaKey %"
PRIx64
"-%"
PRIx64
", dataLen %"
PRIu32
" succeed"
,
SMA_VID
(
pSmaH
->
pSma
),
pDBFile
->
path
,
*
(
int64_t
*
)
smaKey
,
*
(
int64_t
*
)
POINTER_SHIFT
(
smaKey
,
8
),
dataLen
);
smaDebug
(
"vgId:%d insert
t
sma data blocks into %s: smaKey %"
PRIx64
"-%"
PRIx64
", dataLen %"
PRIu32
" succeed"
,
SMA_VID
(
pSmaH
->
pSma
),
pDBFile
->
path
,
*
(
int64_t
*
)
smaKey
,
*
(
int64_t
*
)
POINTER_SHIFT
(
smaKey
,
8
),
dataLen
);
#ifdef _TEST_SMA_PRINT_DEBUG_LOG_
uint32_t
valueSize
=
0
;
...
...
@@ -776,6 +754,8 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query
tdUnRefSmaStat
(
pSma
,
pStat
);
tdInitTSmaFile
(
&
tReadH
,
indexUid
,
querySKey
);
smaDebug
(
"### vgId:%d read from DBF %s days:%d, interval:%"
PRIi64
", storageLevel:%"
PRIi8
" queryKey:%"
PRIi64
,
SMA_VID
(
pSma
),
tReadH
.
dFile
.
path
,
tReadH
.
days
,
tReadH
.
interval
,
tReadH
.
storageLevel
,
querySKey
);
if
(
smaOpenDBF
(
pEnv
->
dbEnv
,
&
tReadH
.
dFile
)
!=
0
)
{
smaWarn
(
"vgId:%d open DBF %s failed since %s"
,
SMA_VID
(
pSma
),
tReadH
.
dFile
.
path
,
tstrerror
(
terrno
));
return
TSDB_CODE_FAILED
;
...
...
@@ -783,7 +763,7 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query
char
smaKey
[
SMA_KEY_LEN
]
=
{
0
};
void
*
pSmaKey
=
&
smaKey
;
int64_t
queryGroupId
=
1
;
int64_t
queryGroupId
=
0
;
tdEncodeTSmaKey
(
queryGroupId
,
querySKey
,
(
void
**
)
&
pSmaKey
);
smaDebug
(
"vgId:%d get sma data from %s: smaKey %"
PRIx64
"-%"
PRIx64
", keyLen %d"
,
SMA_VID
(
pSma
),
...
...
@@ -915,8 +895,8 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde
terrno
=
TSDB_CODE_TDB_NO_SMA_INDEX_IN_META
;
taosHashCleanup
(
pItem
->
expiredWindows
);
taosMemoryFree
(
pItem
);
smaWarn
(
"vgId:%d
update expired window failed for smaIndex %"
PRIi64
" since %s"
,
SMA_VID
(
pSma
),
indexUid
,
tstrerror
(
terrno
));
smaWarn
(
"vgId:%d
set expire window, get tsma meta failed for smaIndex %"
PRIi64
" since %s"
,
SMA_VID
(
pSma
)
,
indexUid
,
tstrerror
(
terrno
));
return
TSDB_CODE_FAILED
;
}
pItem
->
pTSma
=
pTSma
;
...
...
@@ -1021,7 +1001,7 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version)
pSW
=
tdFreeTSmaWrapper
(
pSW
,
false
);
break
;
}
if
(
!
pSW
||
(
pTSma
->
tableUid
!=
msgIter
.
suid
))
{
if
(
!
pSW
||
(
pTSma
&&
(
pTSma
->
tableUid
!=
msgIter
.
suid
)
))
{
if
(
pSW
)
{
pSW
=
tdFreeTSmaWrapper
(
pSW
,
false
);
}
...
...
@@ -1043,6 +1023,7 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version)
interval
.
slidingUnit
=
pTSma
->
slidingUnit
;
}
// TODO: process multiple tsma for one table uid
TSKEY
winSKey
=
taosTimeTruncate
(
TD_ROW_KEY
(
row
),
&
interval
,
interval
.
precision
);
if
(
lastWinSKey
!=
winSKey
)
{
...
...
source/dnode/vnode/src/tsdb/tsdbOpen.c
浏览文件 @
bd5d21b2
...
...
@@ -42,7 +42,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
int
slen
=
0
;
*
ppTsdb
=
NULL
;
slen
=
strlen
(
tfsGetPrimaryPath
(
pVnode
->
pTfs
))
+
strlen
(
pVnode
->
path
)
+
strlen
(
dir
)
+
TSDB_DATA_DIR_LEN
+
3
;
slen
=
strlen
(
tfsGetPrimaryPath
(
pVnode
->
pTfs
))
+
strlen
(
pVnode
->
path
)
+
strlen
(
dir
)
+
3
;
// create handle
pTsdb
=
(
STsdb
*
)
taosMemoryCalloc
(
1
,
sizeof
(
*
pTsdb
)
+
slen
);
...
...
@@ -73,7 +73,8 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
goto
_err
;
}
tsdbDebug
(
"vgId:%d tsdb is opened for %s"
,
TD_VID
(
pVnode
),
pTsdb
->
path
);
tsdbDebug
(
"vgId:%d tsdb is opened for %s, days:%d, keep:%d,%d,%d"
,
TD_VID
(
pVnode
),
pTsdb
->
path
,
pTsdb
->
keepCfg
.
days
,
pTsdb
->
keepCfg
.
keep0
,
pTsdb
->
keepCfg
.
keep1
,
pTsdb
->
keepCfg
.
keep2
);
*
ppTsdb
=
pTsdb
;
return
0
;
...
...
source/dnode/vnode/src/tsdb/tsdbRead.c
浏览文件 @
bd5d21b2
...
...
@@ -372,13 +372,13 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle,
}
if
(
level
==
TSDB_RETENTION_L0
)
{
tsdbDebug
(
"
%p rsma level %d is selected to query"
,
pReadHandle
,
TSDB_RETENTION_L0
);
tsdbDebug
(
"
vgId:%d read handle %p rsma level %d is selected to query"
,
TD_VID
(
pVnode
)
,
pReadHandle
,
TSDB_RETENTION_L0
);
return
VND_RSMA0
(
pVnode
);
}
else
if
(
level
==
TSDB_RETENTION_L1
)
{
tsdbDebug
(
"
%p rsma level %d is selected to query"
,
pReadHandle
,
TSDB_RETENTION_L1
);
tsdbDebug
(
"
vgId:%d read handle %p rsma level %d is selected to query"
,
TD_VID
(
pVnode
)
,
pReadHandle
,
TSDB_RETENTION_L1
);
return
VND_RSMA1
(
pVnode
);
}
else
{
tsdbDebug
(
"
%p rsma level %d is selected to query"
,
pReadHandle
,
TSDB_RETENTION_L2
);
tsdbDebug
(
"
vgId:%d read handle %p rsma level %d is selected to query"
,
TD_VID
(
pVnode
)
,
pReadHandle
,
TSDB_RETENTION_L2
);
return
VND_RSMA2
(
pVnode
);
}
}
...
...
source/dnode/vnode/src/tsdb/tsdbSma.c
浏览文件 @
bd5d21b2
...
...
@@ -876,13 +876,13 @@ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t ke
TXN
*
txn
)
{
SDBFile
*
pDBFile
=
&
pSmaH
->
dFile
;
// TODO: insert sma data blocks into B+Tree(TDB)
// TODO: insert
t
sma data blocks into B+Tree(TDB)
if
(
tsdbSaveSmaToDB
(
pDBFile
,
smaKey
,
keyLen
,
pData
,
dataLen
,
txn
)
!=
0
)
{
tsdbWarn
(
"vgId:%d insert sma data blocks into %s: smaKey %"
PRIx64
"-%"
PRIx64
", dataLen %"
PRIu32
" fail"
,
tsdbWarn
(
"vgId:%d insert
t
sma data blocks into %s: smaKey %"
PRIx64
"-%"
PRIx64
", dataLen %"
PRIu32
" fail"
,
REPO_ID
(
pSmaH
->
pTsdb
),
pDBFile
->
path
,
*
(
int64_t
*
)
smaKey
,
*
(
int64_t
*
)
POINTER_SHIFT
(
smaKey
,
8
),
dataLen
);
return
TSDB_CODE_FAILED
;
}
tsdbDebug
(
"vgId:%d insert sma data blocks into %s: smaKey %"
PRIx64
"-%"
PRIx64
", dataLen %"
PRIu32
" succeed"
,
tsdbDebug
(
"vgId:%d insert
t
sma data blocks into %s: smaKey %"
PRIx64
"-%"
PRIx64
", dataLen %"
PRIu32
" succeed"
,
REPO_ID
(
pSmaH
->
pTsdb
),
pDBFile
->
path
,
*
(
int64_t
*
)
smaKey
,
*
(
int64_t
*
)
POINTER_SHIFT
(
smaKey
,
8
),
dataLen
);
#ifdef _TEST_SMA_PRINT_DEBUG_LOG_
...
...
@@ -1245,7 +1245,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
}
if
(
tsdbInsertTSmaBlocks
(
&
tSmaH
,
&
smaKey
,
SMA_KEY_LEN
,
dataBuf
,
tlen
,
&
pEnv
->
txn
)
!=
0
)
{
tsdbWarn
(
"vgId:%d insert t
S
ma data blocks fail for index %"
PRIi64
", skey %"
PRIi64
", groupId %"
PRIi64
tsdbWarn
(
"vgId:%d insert t
s
ma data blocks fail for index %"
PRIi64
", skey %"
PRIi64
", groupId %"
PRIi64
" since %s"
,
REPO_ID
(
pTsdb
),
indexUid
,
skey
,
groupId
,
tstrerror
(
terrno
));
tsdbSmaEndCommit
(
pEnv
);
...
...
@@ -1253,7 +1253,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
tsdbUnRefSmaStat
(
pTsdb
,
pStat
);
return
TSDB_CODE_FAILED
;
}
tsdbDebug
(
"vgId:%d insert t
S
ma data blocks success for index %"
PRIi64
", skey %"
PRIi64
", groupId %"
PRIi64
,
tsdbDebug
(
"vgId:%d insert t
s
ma data blocks success for index %"
PRIi64
", skey %"
PRIi64
", groupId %"
PRIi64
,
REPO_ID
(
pTsdb
),
indexUid
,
skey
,
groupId
);
// TODO:tsdbEndTSmaCommit();
...
...
source/dnode/vnode/src/vnd/vnodeOpen.c
浏览文件 @
bd5d21b2
...
...
@@ -103,7 +103,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
// open sma
if
(
smaOpen
(
pVnode
))
{
vError
(
"vgId:%d failed to open vnode
tsdb
since %s"
,
TD_VID
(
pVnode
),
tstrerror
(
terrno
));
vError
(
"vgId:%d failed to open vnode
sma
since %s"
,
TD_VID
(
pVnode
),
tstrerror
(
terrno
));
goto
_err
;
}
...
...
source/libs/function/inc/builtinsimpl.h
浏览文件 @
bd5d21b2
...
...
@@ -85,6 +85,11 @@ bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo)
int32_t
spreadFunction
(
SqlFunctionCtx
*
pCtx
);
int32_t
spreadFinalize
(
SqlFunctionCtx
*
pCtx
,
SSDataBlock
*
pBlock
);
bool
getElapsedFuncEnv
(
struct
SFunctionNode
*
pFunc
,
SFuncExecEnv
*
pEnv
);
bool
elapsedFunctionSetup
(
SqlFunctionCtx
*
pCtx
,
SResultRowEntryInfo
*
pResultInfo
);
int32_t
elapsedFunction
(
SqlFunctionCtx
*
pCtx
);
int32_t
elapsedFinalize
(
SqlFunctionCtx
*
pCtx
,
SSDataBlock
*
pBlock
);
bool
getHistogramFuncEnv
(
struct
SFunctionNode
*
pFunc
,
SFuncExecEnv
*
pEnv
);
bool
histogramFunctionSetup
(
SqlFunctionCtx
*
pCtx
,
SResultRowEntryInfo
*
pResultInfo
);
int32_t
histogramFunction
(
SqlFunctionCtx
*
pCtx
);
...
...
source/libs/function/src/builtins.c
浏览文件 @
bd5d21b2
...
...
@@ -226,6 +226,27 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
return
TSDB_CODE_SUCCESS
;
}
static
int32_t
translateElapsed
(
SFunctionNode
*
pFunc
,
char
*
pErrBuf
,
int32_t
len
)
{
int32_t
paraNum
=
LIST_LENGTH
(
pFunc
->
pParameterList
);
if
(
1
!=
paraNum
&&
2
!=
paraNum
)
{
return
invaildFuncParaNumErrMsg
(
pErrBuf
,
len
,
pFunc
->
functionName
);
}
SNode
*
pPara
=
nodesListGetNode
(
pFunc
->
pParameterList
,
0
);
if
(
QUERY_NODE_COLUMN
!=
nodeType
(
pPara
))
{
return
buildFuncErrMsg
(
pErrBuf
,
len
,
TSDB_CODE_FUNC_FUNTION_ERROR
,
"The input parameter of ELAPSED function can only be column"
);
}
uint8_t
paraType
=
((
SExprNode
*
)
nodesListGetNode
(
pFunc
->
pParameterList
,
0
))
->
resType
.
type
;
if
(
TSDB_DATA_TYPE_TIMESTAMP
!=
paraType
)
{
return
invaildFuncParaTypeErrMsg
(
pErrBuf
,
len
,
pFunc
->
functionName
);
}
pFunc
->
node
.
resType
=
(
SDataType
){.
bytes
=
tDataTypes
[
TSDB_DATA_TYPE_DOUBLE
].
bytes
,
.
type
=
TSDB_DATA_TYPE_DOUBLE
};
return
TSDB_CODE_SUCCESS
;
}
static
int32_t
translateLeastSQR
(
SFunctionNode
*
pFunc
,
char
*
pErrBuf
,
int32_t
len
)
{
int32_t
numOfParams
=
LIST_LENGTH
(
pFunc
->
pParameterList
);
if
(
3
!=
numOfParams
)
{
...
...
@@ -794,6 +815,17 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.
processFunc
=
spreadFunction
,
.
finalizeFunc
=
spreadFinalize
},
{
.
name
=
"elapsed"
,
.
type
=
FUNCTION_TYPE_ELAPSED
,
.
classification
=
FUNC_MGT_AGG_FUNC
,
.
dataRequiredFunc
=
statisDataRequired
,
.
translateFunc
=
translateElapsed
,
.
getEnvFunc
=
getElapsedFuncEnv
,
.
initFunc
=
elapsedFunctionSetup
,
.
processFunc
=
elapsedFunction
,
.
finalizeFunc
=
elapsedFinalize
},
{
.
name
=
"last_row"
,
.
type
=
FUNCTION_TYPE_LAST_ROW
,
...
...
source/libs/function/src/builtinsimpl.c
浏览文件 @
bd5d21b2
...
...
@@ -112,6 +112,13 @@ typedef struct SSpreadInfo {
double
max
;
}
SSpreadInfo
;
typedef
struct
SElapsedInfo
{
double
result
;
TSKEY
min
;
TSKEY
max
;
int64_t
timeUnit
;
}
SElapsedInfo
;
typedef
struct
SHistoFuncBin
{
double
lower
;
double
upper
;
...
...
@@ -2494,6 +2501,116 @@ int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
return
functionFinalize
(
pCtx
,
pBlock
);
}
bool
getElapsedFuncEnv
(
SFunctionNode
*
UNUSED_PARAM
(
pFunc
),
SFuncExecEnv
*
pEnv
)
{
pEnv
->
calcMemSize
=
sizeof
(
SElapsedInfo
);
return
true
;
}
bool
elapsedFunctionSetup
(
SqlFunctionCtx
*
pCtx
,
SResultRowEntryInfo
*
pResultInfo
)
{
if
(
!
functionSetup
(
pCtx
,
pResultInfo
))
{
return
false
;
}
SElapsedInfo
*
pInfo
=
GET_ROWCELL_INTERBUF
(
pResultInfo
);
pInfo
->
result
=
0
;
pInfo
->
min
=
MAX_TS_KEY
;
pInfo
->
max
=
0
;
if
(
pCtx
->
numOfParams
==
3
)
{
pInfo
->
timeUnit
=
pCtx
->
param
[
1
].
param
.
i
;
}
else
{
pInfo
->
timeUnit
=
1
;
}
return
true
;
}
int32_t
elapsedFunction
(
SqlFunctionCtx
*
pCtx
)
{
int32_t
numOfElems
=
0
;
// Only the pre-computing information loaded and actual data does not loaded
SInputColumnInfoData
*
pInput
=
&
pCtx
->
input
;
SColumnDataAgg
*
pAgg
=
pInput
->
pColumnDataAgg
[
0
];
SElapsedInfo
*
pInfo
=
GET_ROWCELL_INTERBUF
(
GET_RES_INFO
(
pCtx
));
numOfElems
=
pInput
->
numOfRows
;
//since this is the primary timestamp, no need to exclude NULL values
if
(
numOfElems
==
0
)
{
goto
_elapsed_over
;
}
if
(
pInput
->
colDataAggIsSet
)
{
if
(
pInfo
->
min
==
MAX_TS_KEY
)
{
pInfo
->
min
=
GET_INT64_VAL
(
&
pAgg
->
min
);
pInfo
->
max
=
GET_INT64_VAL
(
&
pAgg
->
max
);
}
else
{
if
(
pCtx
->
order
==
TSDB_ORDER_ASC
)
{
pInfo
->
max
=
GET_INT64_VAL
(
&
pAgg
->
max
);
}
else
{
pInfo
->
min
=
GET_INT64_VAL
(
&
pAgg
->
min
);
}
}
}
else
{
// computing based on the true data block
if
(
0
==
pCtx
->
size
)
{
if
(
pCtx
->
order
==
TSDB_ORDER_DESC
)
{
if
(
pCtx
->
end
.
key
!=
INT64_MIN
)
{
pInfo
->
min
=
pCtx
->
end
.
key
;
}
}
else
{
if
(
pCtx
->
end
.
key
!=
INT64_MIN
)
{
pInfo
->
max
=
pCtx
->
end
.
key
+
1
;
}
}
goto
_elapsed_over
;
}
SColumnInfoData
*
pCol
=
pInput
->
pData
[
0
];
int32_t
start
=
pInput
->
startRowIndex
;
TSKEY
*
ptsList
=
(
int64_t
*
)
colDataGetData
(
pCol
,
start
);
if
(
pCtx
->
order
==
TSDB_ORDER_DESC
)
{
if
(
pCtx
->
start
.
key
==
INT64_MIN
)
{
pInfo
->
max
=
(
pInfo
->
max
<
ptsList
[
pCtx
->
size
-
1
])
?
ptsList
[
pCtx
->
size
-
1
]
:
pInfo
->
max
;
}
else
{
pInfo
->
max
=
pCtx
->
start
.
key
+
1
;
}
if
(
pCtx
->
end
.
key
!=
INT64_MIN
)
{
pInfo
->
min
=
pCtx
->
end
.
key
;
}
else
{
pInfo
->
min
=
ptsList
[
0
];
}
}
else
{
if
(
pCtx
->
start
.
key
==
INT64_MIN
)
{
pInfo
->
min
=
(
pInfo
->
min
>
ptsList
[
0
])
?
ptsList
[
0
]
:
pInfo
->
min
;
}
else
{
pInfo
->
min
=
pCtx
->
start
.
key
;
}
if
(
pCtx
->
end
.
key
!=
INT64_MIN
)
{
pInfo
->
max
=
pCtx
->
end
.
key
+
1
;
}
else
{
pInfo
->
max
=
ptsList
[
pCtx
->
size
-
1
];
}
}
}
_elapsed_over:
// data in the check operation are all null, not output
SET_VAL
(
GET_RES_INFO
(
pCtx
),
numOfElems
,
1
);
return
TSDB_CODE_SUCCESS
;
}
int32_t
elapsedFinalize
(
SqlFunctionCtx
*
pCtx
,
SSDataBlock
*
pBlock
)
{
SElapsedInfo
*
pInfo
=
GET_ROWCELL_INTERBUF
(
GET_RES_INFO
(
pCtx
));
double
result
=
(
double
)
pInfo
->
max
-
(
double
)
pInfo
->
min
;
result
=
(
result
>=
0
)
?
result
:
-
result
;
pInfo
->
result
=
result
/
pInfo
->
timeUnit
;
return
functionFinalize
(
pCtx
,
pBlock
);
}
bool
getHistogramFuncEnv
(
SFunctionNode
*
UNUSED_PARAM
(
pFunc
),
SFuncExecEnv
*
pEnv
)
{
pEnv
->
calcMemSize
=
sizeof
(
SHistoFuncInfo
)
+
HISTOGRAM_MAX_BINS_NUM
*
sizeof
(
SHistoFuncBin
);
return
true
;
...
...
source/libs/function/src/tudf.c
浏览文件 @
bd5d21b2
...
...
@@ -1605,7 +1605,12 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult
return
false
;
}
udfRes
->
interResNum
=
buf
.
numOfResult
;
memcpy
(
udfRes
->
interResBuf
,
buf
.
buf
,
buf
.
bufLen
);
if
(
buf
.
bufLen
<=
session
->
bufSize
)
{
memcpy
(
udfRes
->
interResBuf
,
buf
.
buf
,
buf
.
bufLen
);
}
else
{
fnError
(
"udfc inter buf size %d is greater than function bufSize %d"
,
buf
.
bufLen
,
session
->
bufSize
);
return
false
;
}
freeUdfInterBuf
(
&
buf
);
return
true
;
}
...
...
@@ -1655,7 +1660,12 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
newState
.
numOfResult
=
0
;
}
else
{
udfRes
->
interResNum
=
newState
.
numOfResult
;
memcpy
(
udfRes
->
interResBuf
,
newState
.
buf
,
newState
.
bufLen
);
if
(
newState
.
bufLen
<=
session
->
bufSize
)
{
memcpy
(
udfRes
->
interResBuf
,
newState
.
buf
,
newState
.
bufLen
);
}
else
{
fnError
(
"udfc inter buf size %d is greater than function bufSize %d"
,
newState
.
bufLen
,
session
->
bufSize
);
udfCode
=
TSDB_CODE_UDF_INVALID_BUFSIZE
;
}
}
if
(
newState
.
numOfResult
==
1
||
state
.
numOfResult
==
1
)
{
GET_RES_INFO
(
pCtx
)
->
numOfRes
=
1
;
...
...
@@ -1688,9 +1698,15 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
fnError
(
"udfAggFinalize error. doCallUdfAggFinalize step. udf code:%d"
,
udfCallCode
);
GET_RES_INFO
(
pCtx
)
->
numOfRes
=
0
;
}
else
{
memcpy
(
udfRes
->
finalResBuf
,
resultBuf
.
buf
,
session
->
outputLen
);
udfRes
->
finalResNum
=
resultBuf
.
numOfResult
;
GET_RES_INFO
(
pCtx
)
->
numOfRes
=
udfRes
->
finalResNum
;
if
(
resultBuf
.
bufLen
<=
session
->
outputLen
)
{
memcpy
(
udfRes
->
finalResBuf
,
resultBuf
.
buf
,
session
->
outputLen
);
udfRes
->
finalResNum
=
resultBuf
.
numOfResult
;
GET_RES_INFO
(
pCtx
)
->
numOfRes
=
udfRes
->
finalResNum
;
}
else
{
fnError
(
"udfc inter buf size %d is greater than function output size %d"
,
resultBuf
.
bufLen
,
session
->
outputLen
);
GET_RES_INFO
(
pCtx
)
->
numOfRes
=
0
;
udfCallCode
=
TSDB_CODE_UDF_INVALID_OUTPUT_TYPE
;
}
}
freeUdfInterBuf
(
&
resultBuf
);
...
...
source/libs/function/src/udfd.c
浏览文件 @
bd5d21b2
...
...
@@ -616,9 +616,8 @@ void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
}
void
udfdOnNewConnection
(
uv_stream_t
*
server
,
int
status
)
{
fnDebug
(
"new connection"
);
if
(
status
<
0
)
{
// TODO
fnError
(
"udfd new connection error. code: %s"
,
uv_strerror
(
status
));
return
;
}
...
...
source/libs/index/src/indexComm.c
浏览文件 @
bd5d21b2
...
...
@@ -20,12 +20,13 @@
#include "tcompare.h"
#include "tdataformat.h"
#include "ttypes.h"
#include "tvariant.h"
char
JSON_COLUMN
[]
=
"JSON"
;
char
JSON_VALUE_DELIM
=
'&'
;
char
*
indexInt2str
(
int64_t
val
,
char
*
dst
,
int
radix
)
{
char
buffer
[
65
];
char
buffer
[
65
]
=
{
0
}
;
char
*
p
;
int64_t
new_val
;
uint64_t
uval
=
(
uint64_t
)
val
;
...
...
@@ -74,28 +75,70 @@ static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
return
tCompare
(
func
,
QUERY_GREATER_EQUAL
,
a
,
b
,
type
);
}
TExeCond
tCompare
(
__compar_fn_t
func
,
int8_t
cmptype
,
void
*
a
,
void
*
b
,
int8_t
dtype
)
{
if
(
dtype
==
TSDB_DATA_TYPE_BINARY
||
dtype
==
TSDB_DATA_TYPE_NCHAR
)
{
if
(
dtype
==
TSDB_DATA_TYPE_BINARY
||
dtype
==
TSDB_DATA_TYPE_NCHAR
||
dtype
==
TSDB_DATA_TYPE_VARBINARY
)
{
return
tDoCompare
(
func
,
cmptype
,
a
,
b
);
}
#if 1
int8_t
bytes
=
tDataTypes
[
dtype
].
bytes
;
if
(
bytes
==
1
)
{
if
(
dtype
==
TSDB_DATA_TYPE_TIMESTAMP
)
{
int64_t
va
=
taosStr2int64
(
a
);
int64_t
vb
=
taosStr2int64
(
b
);
return
tDoCompare
(
func
,
cmptype
,
&
va
,
&
vb
);
}
else
if
(
dtype
==
TSDB_DATA_TYPE_BOOL
||
dtype
==
TSDB_DATA_TYPE_UTINYINT
)
{
uint8_t
va
=
taosStr2int64
(
a
);
uint8_t
vb
=
taosStr2int64
(
b
);
return
tDoCompare
(
func
,
cmptype
,
&
va
,
&
vb
);
}
else
if
(
dtype
==
TSDB_DATA_TYPE_TINYINT
)
{
int8_t
va
=
taosStr2int64
(
a
);
int8_t
vb
=
taosStr2int64
(
b
);
return
tDoCompare
(
func
,
cmptype
,
&
va
,
&
vb
);
}
else
if
(
bytes
==
2
)
{
}
else
if
(
dtype
==
TSDB_DATA_TYPE_SMALLINT
)
{
int16_t
va
=
taosStr2int64
(
a
);
int16_t
vb
=
taosStr2int64
(
b
);
return
tDoCompare
(
func
,
cmptype
,
&
va
,
&
vb
);
}
else
if
(
bytes
==
4
)
{
}
else
if
(
dtype
==
TSDB_DATA_TYPE_USMALLINT
)
{
uint16_t
va
=
taosStr2int64
(
a
);
uint16_t
vb
=
taosStr2int64
(
b
);
return
tDoCompare
(
func
,
cmptype
,
&
va
,
&
vb
);
}
else
if
(
dtype
==
TSDB_DATA_TYPE_INT
)
{
int32_t
va
=
taosStr2int64
(
a
);
int32_t
vb
=
taosStr2int64
(
b
);
return
tDoCompare
(
func
,
cmptype
,
&
va
,
&
vb
);
}
else
{
}
else
if
(
dtype
==
TSDB_DATA_TYPE_UINT
)
{
uint32_t
va
=
taosStr2int64
(
a
);
uint32_t
vb
=
taosStr2int64
(
b
);
return
tDoCompare
(
func
,
cmptype
,
&
va
,
&
vb
);
}
else
if
(
dtype
==
TSDB_DATA_TYPE_BIGINT
)
{
int64_t
va
=
taosStr2int64
(
a
);
int64_t
vb
=
taosStr2int64
(
b
);
return
tDoCompare
(
func
,
cmptype
,
&
va
,
&
vb
);
}
else
if
(
dtype
==
TSDB_DATA_TYPE_UBIGINT
)
{
uint64_t
va
,
vb
;
if
(
0
!=
toUInteger
(
a
,
strlen
(
a
),
10
,
&
va
)
||
0
!=
toUInteger
(
b
,
strlen
(
b
),
10
,
&
vb
))
{
return
CONTINUE
;
}
return
tDoCompare
(
func
,
cmptype
,
&
va
,
&
vb
);
}
else
if
(
dtype
==
TSDB_DATA_TYPE_FLOAT
)
{
float
va
=
strtod
(
a
,
NULL
);
if
(
errno
==
ERANGE
&&
va
==
-
1
)
{
return
CONTINUE
;
}
float
vb
=
strtod
(
b
,
NULL
);
if
(
errno
==
ERANGE
&&
va
==
-
1
)
{
return
CONTINUE
;
}
return
tDoCompare
(
func
,
cmptype
,
&
va
,
&
vb
);
}
else
if
(
dtype
==
TSDB_DATA_TYPE_DOUBLE
)
{
double
va
=
strtod
(
a
,
NULL
);
if
(
errno
==
ERANGE
&&
va
==
-
1
)
{
return
CONTINUE
;
}
double
vb
=
strtod
(
b
,
NULL
);
if
(
errno
==
ERANGE
&&
va
==
-
1
)
{
return
CONTINUE
;
}
return
tDoCompare
(
func
,
cmptype
,
&
va
,
&
vb
);
}
assert
(
0
);
#endif
}
TExeCond
tDoCompare
(
__compar_fn_t
func
,
int8_t
comparType
,
void
*
a
,
void
*
b
)
{
...
...
@@ -248,20 +291,16 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
break
;
}
case
TSDB_DATA_TYPE_VARCHAR
:
{
// TSDB_DATA_TYPE_BINARY
#if 1
tlen
=
taosEncodeBinary
(
NULL
,
src
,
strlen
(
src
));
*
dst
=
taosMemoryCalloc
(
1
,
tlen
+
1
);
tlen
=
taosEncodeBinary
(
dst
,
src
,
strlen
(
src
));
break
;
#endif
}
case
TSDB_DATA_TYPE_VARBINARY
:
#if 1
tlen
=
taosEncodeBinary
(
NULL
,
src
,
strlen
(
src
));
*
dst
=
taosMemoryCalloc
(
1
,
tlen
+
1
);
tlen
=
taosEncodeBinary
(
dst
,
src
,
strlen
(
src
));
break
;
#endif
default:
TASSERT
(
0
);
break
;
...
...
@@ -271,87 +310,73 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
return
tlen
;
}
int32_t
indexConvertDataToStr
(
void
*
src
,
int8_t
type
,
void
**
dst
)
{
int
tlen
=
tDataTypes
[
type
].
bytes
;
int
tlen
=
tDataTypes
[
type
].
bytes
;
int32_t
bufSize
=
64
;
switch
(
type
)
{
case
TSDB_DATA_TYPE_TIMESTAMP
:
*
dst
=
taosMemoryCalloc
(
1
,
sizeof
(
int64_t
)
+
1
);
*
dst
=
taosMemoryCalloc
(
1
,
bufSize
+
1
);
indexInt2str
(
*
(
int64_t
*
)
src
,
*
dst
,
-
1
);
break
;
case
TSDB_DATA_TYPE_BOOL
:
case
TSDB_DATA_TYPE_UTINYINT
:
// tlen = taosEncodeFixedU8(NULL, *(uint8_t*)src);
//*dst = taosMemoryCalloc(1, tlen + 1);
// tlen = taosEncodeFixedU8(dst, *(uint8_t*)src);
*
dst
=
taosMemoryCalloc
(
1
,
sizeof
(
int64_t
)
+
1
);
*
dst
=
taosMemoryCalloc
(
1
,
bufSize
+
1
);
indexInt2str
(
*
(
uint8_t
*
)
src
,
*
dst
,
1
);
break
;
case
TSDB_DATA_TYPE_TINYINT
:
*
dst
=
taosMemoryCalloc
(
1
,
sizeof
(
int64_t
)
+
1
);
*
dst
=
taosMemoryCalloc
(
1
,
bufSize
+
1
);
indexInt2str
(
*
(
int8_t
*
)
src
,
*
dst
,
1
);
break
;
case
TSDB_DATA_TYPE_SMALLINT
:
*
dst
=
taosMemoryCalloc
(
1
,
sizeof
(
int64_t
)
+
1
);
*
dst
=
taosMemoryCalloc
(
1
,
bufSize
+
1
);
indexInt2str
(
*
(
int16_t
*
)
src
,
*
dst
,
-
1
);
break
;
case
TSDB_DATA_TYPE_USMALLINT
:
*
dst
=
taosMemoryCalloc
(
1
,
sizeof
(
int64_t
)
+
1
);
*
dst
=
taosMemoryCalloc
(
1
,
bufSize
+
1
);
indexInt2str
(
*
(
uint16_t
*
)
src
,
*
dst
,
-
1
);
break
;
case
TSDB_DATA_TYPE_INT
:
*
dst
=
taosMemoryCalloc
(
1
,
sizeof
(
int64_t
)
+
1
);
*
dst
=
taosMemoryCalloc
(
1
,
bufSize
+
1
);
indexInt2str
(
*
(
int32_t
*
)
src
,
*
dst
,
-
1
);
break
;
case
TSDB_DATA_TYPE_FLOAT
:
tlen
=
taosEncodeBinary
(
NULL
,
src
,
sizeof
(
float
));
*
dst
=
taosMemoryCalloc
(
1
,
tlen
+
1
);
tlen
=
taosEncodeBinary
(
dst
,
src
,
sizeof
(
float
));
*
dst
=
(
char
*
)
*
dst
-
tlen
;
break
;
case
TSDB_DATA_TYPE_UINT
:
*
dst
=
taosMemoryCalloc
(
1
,
sizeof
(
int64_t
)
+
1
);
*
dst
=
taosMemoryCalloc
(
1
,
bufSize
+
1
);
indexInt2str
(
*
(
uint32_t
*
)
src
,
*
dst
,
1
);
break
;
case
TSDB_DATA_TYPE_BIGINT
:
*
dst
=
taosMemoryCalloc
(
1
,
sizeof
(
int64_t
)
+
1
);
indexInt2str
(
*
(
int64_t
*
)
src
,
*
dst
,
1
);
break
;
case
TSDB_DATA_TYPE_DOUBLE
:
tlen
=
taosEncodeBinary
(
NULL
,
src
,
sizeof
(
double
));
*
dst
=
taosMemoryCalloc
(
1
,
tlen
+
1
);
tlen
=
taosEncodeBinary
(
dst
,
src
,
sizeof
(
double
));
*
dst
=
(
char
*
)
*
dst
-
tlen
;
*
dst
=
taosMemoryCalloc
(
1
,
bufSize
+
1
);
sprintf
(
*
dst
,
"%"
PRIu64
,
*
(
uint64_t
*
)
src
);
break
;
case
TSDB_DATA_TYPE_UBIGINT
:
assert
(
0
);
*
dst
=
taosMemoryCalloc
(
1
,
sizeof
(
int64_t
)
+
1
);
*
dst
=
taosMemoryCalloc
(
1
,
bufSize
+
1
);
indexInt2str
(
*
(
uint64_t
*
)
src
,
*
dst
,
1
);
case
TSDB_DATA_TYPE_FLOAT
:
*
dst
=
taosMemoryCalloc
(
1
,
bufSize
+
1
);
sprintf
(
*
dst
,
"%.9lf"
,
*
(
float
*
)
src
);
break
;
case
TSDB_DATA_TYPE_DOUBLE
:
*
dst
=
taosMemoryCalloc
(
1
,
bufSize
+
1
);
sprintf
(
*
dst
,
"%.9lf"
,
*
(
double
*
)
src
);
break
;
case
TSDB_DATA_TYPE_NCHAR
:
{
tlen
=
taosEncodeBinary
(
NULL
,
varDataVal
(
src
),
varDataLen
(
src
));
*
dst
=
taosMemoryCalloc
(
1
,
tlen
+
1
);
tlen
=
taosEncodeBinary
(
dst
,
varDataVal
(
src
),
varDataLen
(
src
));
*
dst
=
(
char
*
)
*
dst
-
tlen
;
*
dst
=
*
dst
-
tlen
;
break
;
}
case
TSDB_DATA_TYPE_VARCHAR
:
{
// TSDB_DATA_TYPE_BINARY
#if 1
tlen
=
taosEncodeBinary
(
NULL
,
src
,
strlen
(
src
));
*
dst
=
taosMemoryCalloc
(
1
,
tlen
+
1
);
tlen
=
taosEncodeBinary
(
dst
,
src
,
strlen
(
src
));
*
dst
=
(
char
*
)
*
dst
-
tlen
;
*
dst
=
(
char
*
)
*
dst
-
tlen
;
break
;
#endif
}
case
TSDB_DATA_TYPE_VARBINARY
:
#if 1
tlen
=
taosEncodeBinary
(
NULL
,
src
,
strlen
(
src
));
*
dst
=
taosMemoryCalloc
(
1
,
tlen
+
1
);
tlen
=
taosEncodeBinary
(
dst
,
src
,
strlen
(
src
));
*
dst
=
(
char
*
)
*
dst
-
tlen
;
*
dst
=
(
char
*
)
*
dst
-
tlen
;
break
;
#endif
default:
TASSERT
(
0
);
break
;
...
...
source/libs/index/src/indexTfile.c
浏览文件 @
bd5d21b2
...
...
@@ -20,6 +20,7 @@ p *
#include "indexFstCountingWriter.h"
#include "indexUtil.h"
#include "taosdef.h"
#include "taoserror.h"
#include "tcoding.h"
#include "tcompare.h"
...
...
@@ -472,16 +473,16 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
int32_t
sz
=
0
;
char
*
ch
=
(
char
*
)
fstSliceData
(
s
,
&
sz
);
char
*
tmp
=
taosMemoryCalloc
(
1
,
sz
+
1
);
memcpy
(
tmp
,
ch
,
sz
);
//
char* tmp = taosMemoryCalloc(1, sz + 1);
//
memcpy(tmp, ch, sz);
if
(
0
!=
strncmp
(
tmp
,
p
,
skip
))
{
if
(
0
!=
strncmp
(
ch
,
p
,
skip
))
{
swsResultDestroy
(
rt
);
taosMemoryFree
(
tmp
);
//
taosMemoryFree(tmp);
break
;
}
TExeCond
cond
=
cmpFn
(
tmp
+
skip
,
tem
->
colVal
,
INDEX_TYPE_GET_TYPE
(
tem
->
colType
));
TExeCond
cond
=
cmpFn
(
ch
+
skip
,
tem
->
colVal
,
INDEX_TYPE_GET_TYPE
(
tem
->
colType
));
if
(
MATCH
==
cond
)
{
tfileReaderLoadTableIds
((
TFileReader
*
)
reader
,
rt
->
out
.
out
,
tr
->
total
);
...
...
@@ -490,7 +491,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
swsResultDestroy
(
rt
);
break
;
}
taosMemoryFree
(
tmp
);
//
taosMemoryFree(tmp);
swsResultDestroy
(
rt
);
}
streamWithStateDestroy
(
st
);
...
...
@@ -533,10 +534,12 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c
tfileGenFileFullName
(
fullname
,
path
,
suid
,
colName
,
version
);
WriterCtx
*
wc
=
writerCtxCreate
(
TFile
,
fullname
,
true
,
1024
*
1024
*
1024
);
indexInfo
(
"open read file name:%s, file size: %d"
,
wc
->
file
.
buf
,
wc
->
file
.
size
);
if
(
wc
==
NULL
)
{
terrno
=
TAOS_SYSTEM_ERROR
(
errno
);
indexError
(
"failed to open readonly file: %s, reason: %s"
,
fullname
,
terrstr
());
return
NULL
;
}
indexInfo
(
"open read file name:%s, file size: %d"
,
wc
->
file
.
buf
,
wc
->
file
.
size
);
TFileReader
*
reader
=
tfileReaderCreate
(
wc
);
return
reader
;
...
...
@@ -613,9 +616,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
if
(
tfileWriteData
(
tw
,
v
)
!=
0
)
{
indexError
(
"failed to write data: %s, offset: %d len: %d"
,
v
->
colVal
,
v
->
offset
,
(
int
)
taosArrayGetSize
(
v
->
tableId
));
// printf("write faile\n");
}
else
{
// printf("write sucee\n");
// indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset,
// (int)taosArrayGetSize(v->tableId));
...
...
source/libs/index/test/jsonUT.cc
浏览文件 @
bd5d21b2
...
...
@@ -553,7 +553,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) {
float
val
=
2.0
;
std
::
string
colName
(
"test1"
);
for
(
int
i
=
0
;
i
<
1000
;
i
++
)
{
WriteData
(
index
,
colName
,
TSDB_DATA_TYPE_FLOAT
,
&
val
,
sizeof
(
val
),
i
);
WriteData
(
index
,
colName
,
TSDB_DATA_TYPE_FLOAT
,
&
val
,
sizeof
(
val
),
i
+
1000
);
}
}
{
...
...
source/libs/tdb/src/db/tdbPCache.c
浏览文件 @
bd5d21b2
...
...
@@ -15,10 +15,10 @@
#include "tdbInt.h"
struct
SPCache
{
int
pageSize
;
int
cacheSize
;
int
szPage
;
int
nPages
;
SPage
**
aPage
;
tdb_mutex_t
mutex
;
SPage
*
pList
;
int
nFree
;
SPage
*
pFree
;
int
nPage
;
...
...
@@ -52,13 +52,14 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
void
*
pPtr
;
SPage
*
pPgHdr
;
pCache
=
(
SPCache
*
)
tdbOsCalloc
(
1
,
sizeof
(
*
pCache
));
pCache
=
(
SPCache
*
)
tdbOsCalloc
(
1
,
sizeof
(
*
pCache
)
+
sizeof
(
SPage
*
)
*
cacheSize
);
if
(
pCache
==
NULL
)
{
return
-
1
;
}
pCache
->
pageSize
=
pageSize
;
pCache
->
cacheSize
=
cacheSize
;
pCache
->
szPage
=
pageSize
;
pCache
->
nPages
=
cacheSize
;
pCache
->
aPage
=
(
SPage
**
)
&
pCache
[
1
];
if
(
tdbPCacheOpenImpl
(
pCache
)
<
0
)
{
tdbOsFree
(
pCache
);
...
...
@@ -84,7 +85,7 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
pPage
=
tdbPCacheFetchImpl
(
pCache
,
pPgid
,
pTxn
);
if
(
pPage
)
{
TDB_REF_PAGE
(
pPage
);
tdbRefPage
(
pPage
);
}
tdbPCacheUnlock
(
pCache
);
...
...
@@ -97,7 +98,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
ASSERT
(
pTxn
);
nRef
=
TDB_UNREF_PAGE
(
pPage
);
nRef
=
tdbUnrefPage
(
pPage
);
ASSERT
(
nRef
>=
0
);
if
(
nRef
==
0
)
{
...
...
@@ -105,7 +106,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
// test the nRef again to make sure
// it is safe th handle the page
nRef
=
TDB_GET_PAGE_REF
(
pPage
);
nRef
=
tdbGetPageRef
(
pPage
);
if
(
nRef
==
0
)
{
if
(
pPage
->
isLocal
)
{
tdbPCacheUnpinPage
(
pCache
,
pPage
);
...
...
@@ -123,7 +124,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
}
}
int
tdbPCacheGetPageSize
(
SPCache
*
pCache
)
{
return
pCache
->
pageSiz
e
;
}
int
tdbPCacheGetPageSize
(
SPCache
*
pCache
)
{
return
pCache
->
szPag
e
;
}
static
SPage
*
tdbPCacheFetchImpl
(
SPCache
*
pCache
,
const
SPgid
*
pPgid
,
TXN
*
pTxn
)
{
int
ret
=
0
;
...
...
@@ -168,7 +169,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
// 4. Try a create new page
if
(
!
pPage
)
{
ret
=
tdbPageCreate
(
pCache
->
pageSiz
e
,
&
pPage
,
pTxn
->
xMalloc
,
pTxn
->
xArg
);
ret
=
tdbPageCreate
(
pCache
->
szPag
e
,
&
pPage
,
pTxn
->
xMalloc
,
pTxn
->
xArg
);
if
(
ret
<
0
)
{
// TODO
ASSERT
(
0
);
...
...
@@ -178,7 +179,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
// init the page fields
pPage
->
isAnchor
=
0
;
pPage
->
isLocal
=
0
;
TDB_INIT_PAGE_REF
(
pPage
);
pPage
->
nRef
=
0
;
pPage
->
id
=
-
1
;
}
// 5. Page here are just created from a free list
...
...
@@ -212,20 +214,25 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
}
static
void
tdbPCachePinPage
(
SPCache
*
pCache
,
SPage
*
pPage
)
{
if
(
!
PAGE_IS_PINNED
(
pPage
))
{
if
(
pPage
->
pLruNext
!=
NULL
)
{
ASSERT
(
tdbGetPageRef
(
pPage
)
==
0
);
pPage
->
pLruPrev
->
pLruNext
=
pPage
->
pLruNext
;
pPage
->
pLruNext
->
pLruPrev
=
pPage
->
pLruPrev
;
pPage
->
pLruNext
=
NULL
;
pCache
->
nRecyclable
--
;
tdbTrace
(
"pin page %d"
,
pPage
->
id
);
}
}
static
void
tdbPCacheUnpinPage
(
SPCache
*
pCache
,
SPage
*
pPage
)
{
i32
nRef
;
ASSERT
(
pPage
->
isLocal
);
ASSERT
(
!
pPage
->
isDirty
);
ASSERT
(
TDB_GET_PAGE_REF
(
pPage
)
==
0
);
ASSERT
(
tdbGetPageRef
(
pPage
)
==
0
);
ASSERT
(
pPage
->
pLruNext
==
NULL
);
...
...
@@ -235,6 +242,8 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) {
pCache
->
lru
.
pLruNext
=
pPage
;
pCache
->
nRecyclable
++
;
tdbTrace
(
"unpin page %d"
,
pPage
->
id
);
}
static
void
tdbPCacheRemovePageFromHash
(
SPCache
*
pCache
,
SPage
*
pPage
)
{
...
...
@@ -248,6 +257,8 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) {
*
ppPage
=
pPage
->
pHashNext
;
pCache
->
nPage
--
;
tdbTrace
(
"remove page %d to hash"
,
pPage
->
id
);
}
static
void
tdbPCacheAddPageToHash
(
SPCache
*
pCache
,
SPage
*
pPage
)
{
...
...
@@ -259,6 +270,8 @@ static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) {
pCache
->
pgHash
[
h
]
=
pPage
;
pCache
->
nPage
++
;
tdbTrace
(
"add page %d to hash"
,
pPage
->
id
);
}
static
int
tdbPCacheOpenImpl
(
SPCache
*
pCache
)
{
...
...
@@ -272,8 +285,8 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
// Open the free list
pCache
->
nFree
=
0
;
pCache
->
pFree
=
NULL
;
for
(
int
i
=
0
;
i
<
pCache
->
cacheSize
;
i
++
)
{
ret
=
tdbPageCreate
(
pCache
->
pageSiz
e
,
&
pPage
,
tdbDefaultMalloc
,
NULL
);
for
(
int
i
=
0
;
i
<
pCache
->
nPages
;
i
++
)
{
ret
=
tdbPageCreate
(
pCache
->
szPag
e
,
&
pPage
,
tdbDefaultMalloc
,
NULL
);
if
(
ret
<
0
)
{
// TODO: handle error
return
-
1
;
...
...
@@ -282,7 +295,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
// pPage->pgid = 0;
pPage
->
isAnchor
=
0
;
pPage
->
isLocal
=
1
;
TDB_INIT_PAGE_REF
(
pPage
)
;
pPage
->
nRef
=
0
;
pPage
->
pHashNext
=
NULL
;
pPage
->
pLruNext
=
NULL
;
pPage
->
pLruPrev
=
NULL
;
...
...
@@ -294,13 +307,13 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
pCache
->
nFree
++
;
// add to local list
pPage
->
pCacheNext
=
pCache
->
pList
;
pCache
->
pList
=
pPage
;
pPage
->
id
=
i
;
pCache
->
aPage
[
i
]
=
pPage
;
}
// Open the hash table
pCache
->
nPage
=
0
;
pCache
->
nHash
=
pCache
->
cacheSize
<
8
?
8
:
pCache
->
cacheSize
;
pCache
->
nHash
=
pCache
->
nPages
<
8
?
8
:
pCache
->
nPages
;
pCache
->
pgHash
=
(
SPage
**
)
tdbOsCalloc
(
pCache
->
nHash
,
sizeof
(
SPage
*
));
if
(
pCache
->
pgHash
==
NULL
)
{
// TODO
...
...
@@ -317,11 +330,11 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
}
static
int
tdbPCacheCloseImpl
(
SPCache
*
pCache
)
{
SPage
*
pPage
;
for
(
pPage
=
pCache
->
pList
;
pPage
;
pPage
=
pCache
->
pList
)
{
pCache
->
pList
=
pPage
->
pCacheNext
;
tdbPageDestroy
(
pPage
,
tdbDefaultFree
,
NULL
);
for
(
i32
iPage
=
0
;
iPage
<
pCache
->
nPages
;
iPage
++
)
{
if
(
pCache
->
aPage
[
iPage
])
{
tdbPageDestroy
(
pCache
->
aPage
[
iPage
],
tdbDefaultFree
,
NULL
);
pCache
->
aPage
[
iPage
]
=
NULL
;
}
}
tdbOsFree
(
pCache
->
pgHash
);
...
...
source/libs/tdb/src/db/tdbPager.c
浏览文件 @
bd5d21b2
...
...
@@ -149,7 +149,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
if
(
pPage
->
isDirty
)
return
0
;
// ref page one more time so the page will not be release
TDB_REF_PAGE
(
pPage
);
tdbRefPage
(
pPage
);
// Set page as dirty
pPage
->
isDirty
=
1
;
...
...
source/libs/tdb/src/inc/tdbInt.h
浏览文件 @
bd5d21b2
...
...
@@ -18,10 +18,23 @@
#include "tdb.h"
#include "tlog.h"
#ifdef __cplusplus
extern
"C"
{
#endif
// clang-format off
extern
int32_t
tdbDebugFlag
;
#define tdbFatal(...) do { if (tdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0)
#define tdbError(...) do { if (tdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0)
#define tdbWarn(...) do { if (tdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0)
#define tdbInfo(...) do { if (tdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0)
#define tdbDebug(...) do { if (tdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", DEBUG_DEBUG, tdbDebugFlag, __VA_ARGS__); }} while(0)
#define tdbTrace(...) do { if (tdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", DEBUG_TRACE, tdbDebugFlag, __VA_ARGS__); }} while(0)
// clang-format on
typedef
int8_t
i8
;
typedef
int16_t
i16
;
typedef
int32_t
i32
;
...
...
@@ -161,25 +174,21 @@ void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn);
int
tdbPagerAllocPage
(
SPager
*
pPager
,
SPgno
*
ppgno
);
// tdbPCache.c ====================================
#define TDB_PCACHE_PAGE \
u8
isAnchor;
\
u8
isLocal;
\
u8
isDirty;
\
i32 nRef;
\
SPage *pCacheNext;
\
SPage
*pFreeNext;
\
SPage
*pHashNext;
\
SPage
*pLruNext;
\
SPage
*pLruPrev;
\
SPage
*pDirtyNext;
\
SPager
*pPager;
\
SPgid pgid;
#define TDB_PCACHE_PAGE
\
u8
isAnchor;
\
u8
isLocal;
\
u8
isDirty;
\
volatile i32 nRef;
\
i32 id;
\
SPage
*pFreeNext;
\
SPage
*pHashNext;
\
SPage
*pLruNext;
\
SPage
*pLruPrev;
\
SPage
*pDirtyNext;
\
SPager
*pPager;
\
SPgid
pgid;
// For page ref
#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0)
#define TDB_REF_PAGE(pPage) atomic_add_fetch_32(&((pPage)->nRef), 1)
#define TDB_UNREF_PAGE(pPage) atomic_sub_fetch_32(&((pPage)->nRef), 1)
#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef))
int
tdbPCacheOpen
(
int
pageSize
,
int
cacheSize
,
SPCache
**
ppCache
);
int
tdbPCacheClose
(
SPCache
*
pCache
);
...
...
@@ -246,6 +255,20 @@ struct SPage {
TDB_PCACHE_PAGE
};
static
inline
i32
tdbRefPage
(
SPage
*
pPage
)
{
i32
nRef
=
atomic_add_fetch_32
(
&
((
pPage
)
->
nRef
),
1
);
tdbTrace
(
"ref page %d, nRef %d"
,
pPage
->
id
,
nRef
);
return
nRef
;
}
static
inline
i32
tdbUnrefPage
(
SPage
*
pPage
)
{
i32
nRef
=
atomic_sub_fetch_32
(
&
((
pPage
)
->
nRef
),
1
);
tdbTrace
(
"unref page %d, nRef %d"
,
pPage
->
id
,
nRef
);
return
nRef
;
}
#define tdbGetPageRef(pPage) atomic_load_32(&((pPage)->nRef))
// For page lock
#define P_LOCK_SUCC 0
#define P_LOCK_BUSY 1
...
...
source/util/src/terror.c
浏览文件 @
bd5d21b2
...
...
@@ -470,6 +470,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure")
TAOS_DEFINE_ERROR
(
TSDB_CODE_UDF_INVALID_STATE
,
"udf invalid state"
)
TAOS_DEFINE_ERROR
(
TSDB_CODE_UDF_INVALID_INPUT
,
"udf invalid function input"
)
TAOS_DEFINE_ERROR
(
TSDB_CODE_UDF_NO_FUNC_HANDLE
,
"udf no function handle"
)
TAOS_DEFINE_ERROR
(
TSDB_CODE_UDF_INVALID_BUFSIZE
,
"udf invalid bufsize"
)
TAOS_DEFINE_ERROR
(
TSDB_CODE_UDF_INVALID_OUTPUT_TYPE
,
"udf invalid output type"
)
//schemaless
TAOS_DEFINE_ERROR
(
TSDB_CODE_SML_INVALID_PROTOCOL_TYPE
,
"Invalid line protocol type"
)
TAOS_DEFINE_ERROR
(
TSDB_CODE_SML_INVALID_PRECISION_TYPE
,
"Invalid timestamp precision type"
)
...
...
source/util/src/tlog.c
浏览文件 @
bd5d21b2
...
...
@@ -14,8 +14,8 @@
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tlog.h"
#include "os.h"
#include "tutil.h"
#define LOG_MAX_LINE_SIZE (1024)
...
...
@@ -90,6 +90,7 @@ int32_t qDebugFlag = 131;
int32_t
wDebugFlag
=
135
;
int32_t
sDebugFlag
=
135
;
int32_t
tsdbDebugFlag
=
131
;
int32_t
tdbDebugFlag
=
131
;
int32_t
tqDebugFlag
=
135
;
int32_t
fsDebugFlag
=
135
;
int32_t
metaDebugFlag
=
135
;
...
...
tests/system-test/2-query/union.py
浏览文件 @
bd5d21b2
...
...
@@ -35,53 +35,26 @@ class TDTestCase:
for
char_col
in
CHAR_COL
:
query_condition
.
extend
(
(
f
"
{
tbname
}
.
{
char_col
}
"
,
f
"upper(
{
tbname
}
.
{
char_col
}
)"
,
f
"char_length(
{
tbname
}
.
{
char_col
}
)"
,
f
"concat(
{
tbname
}
.
{
char_col
}
,
{
tbname
}
.
{
char_col
}
)"
,
f
"concat_ws( '_',
{
tbname
}
.
{
char_col
}
,
{
tbname
}
.
{
char_col
}
)"
,
f
"length(
{
tbname
}
.
{
char_col
}
)"
,
f
"lower(
{
tbname
}
.
{
char_col
}
)"
,
f
"ltrim(
{
tbname
}
.
{
char_col
}
)"
,
f
"rtrim(
{
tbname
}
.
{
char_col
}
)"
,
f
"substr(
{
tbname
}
.
{
char_col
}
, 1 )"
,
f
"count(
{
tbname
}
.
{
char_col
}
)"
,
f
"cast(
{
tbname
}
.
{
char_col
}
as nchar(3) )"
,
f
"cast(
{
tbname
}
.
{
char_col
}
as nchar(8) )"
,
)
)
query_condition
.
extend
(
f
"cast(
{
tbname
}
.
{
un_char_col
}
as binary(16) ) "
for
un_char_col
in
NUM_COL
)
query_condition
.
extend
(
f
"cast(
{
tbname
}
.
{
char_col
}
+
{
tbname
}
.
{
char_col_2
}
as binary(32) ) "
for
char_col_2
in
CHAR_COL
)
query_condition
.
extend
(
f
"cast(
{
tbname
}
.
{
char_col
}
+
{
tbname
}
.
{
un_char_col
}
as binary(32) ) "
for
un_char_col
in
NUM_COL
)
for
num_col
in
NUM_COL
:
query_condition
.
extend
(
(
f
"
{
tbname
}
.
{
num_col
}
"
,
f
"ceil(
{
tbname
}
.
{
num_col
}
)"
,
f
"abs(
{
tbname
}
.
{
num_col
}
)"
,
f
"acos(
{
tbname
}
.
{
num_col
}
)"
,
f
"asin(
{
tbname
}
.
{
num_col
}
)"
,
f
"atan(
{
tbname
}
.
{
num_col
}
)"
,
f
"cos(
{
tbname
}
.
{
num_col
}
)"
,
f
"floor(
{
tbname
}
.
{
num_col
}
)"
,
f
"log(
{
tbname
}
.
{
num_col
}
,
{
tbname
}
.
{
num_col
}
)"
,
f
"sin(
{
tbname
}
.
{
num_col
}
)"
,
f
"sqrt(
{
tbname
}
.
{
num_col
}
)"
,
f
"tan(
{
tbname
}
.
{
num_col
}
)"
,
f
"round(
{
tbname
}
.
{
num_col
}
)"
,
f
"max(
{
tbname
}
.
{
num_col
}
)"
,
f
"sum(
{
tbname
}
.
{
num_col
}
)"
,
f
"count(
{
tbname
}
.
{
num_col
}
)"
,
f
"min(
{
tbname
}
.
{
num_col
}
)"
,
)
)
query_condition
.
extend
(
f
"
{
tbname
}
.
{
num_col
}
+
{
tbname
}
.
{
num_col_2
}
"
for
num_col_2
in
NUM_COL
)
query_condition
.
extend
(
f
"
{
tbname
}
.
{
num_col
}
+
{
tbname
}
.
{
char_col
}
"
for
char_col
in
CHAR_COL
)
query_condition
.
extend
(
(
''' "test1234!@#$%^&*():'><?/.,][}{" '''
,
''' "test12" '''
,
# 1010,
)
...
...
@@ -208,8 +181,8 @@ class TDTestCase:
)
)
return
filter
(
None
,
sqls
)
#
return list(filter(None, sqls))
#
return filter(None, sqls)
return
list
(
filter
(
None
,
sqls
))
def
__get_type
(
self
,
col
):
if
tdSql
.
cursor
.
istype
(
col
,
"BOOL"
):
...
...
@@ -245,34 +218,55 @@ class TDTestCase:
def
union_check
(
self
):
sqls
=
self
.
sql_list
()
for
sql1
in
sqls
:
tdSql
.
query
(
sql
1
)
for
i
in
range
(
len
(
sqls
))
:
tdSql
.
query
(
sql
s
[
i
]
)
res1_type
=
self
.
__get_type
(
0
)
for
sql2
in
sqls
:
tdSql
.
query
(
sql2
)
union_type
=
False
for
j
in
range
(
len
(
sqls
[
i
:])):
tdSql
.
query
(
sqls
[
j
+
i
])
order_union_type
=
False
rev_order_type
=
False
all_union_type
=
False
res2_type
=
self
.
__get_type
(
0
)
if
res1_type
in
(
"BIGINT"
,
"NCHAR"
):
union_type
=
True
elif
res2_type
==
res1_type
:
union_type
=
True
if
res2_type
==
res1_type
:
all_union_type
=
True
elif
res1_type
in
(
"BIGINT"
,
"NCHAR"
)
and
res2_type
in
(
"BIGINT"
,
"NCHAR"
):
all_union_type
=
True
elif
res1_type
in
(
"BIGINT"
,
"NCHAR"
):
order_union_type
=
True
elif
res2_type
in
(
"BIGINT"
,
"NCHAR"
):
rev_order_type
=
True
elif
res1_type
==
"TIMESAMP"
and
res2_type
not
in
(
"BINARY"
,
"NCHAR"
):
union_type
=
True
order_union_type
=
True
elif
res2_type
==
"TIMESAMP"
and
res1_type
not
in
(
"BINARY"
,
"NCHAR"
):
rev_order_type
=
True
elif
res1_type
==
"BINARY"
and
res2_type
!=
"NCHAR"
:
union_type
=
True
order_union_type
=
True
elif
res2_type
==
"BINARY"
and
res1_type
!=
"NCHAR"
:
rev_order_type
=
True
if
union_type
:
tdSql
.
query
(
f
"
{
sql1
}
union
{
sql2
}
"
)
if
all_union_type
:
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
tdSql
.
query
(
f
"
{
sql1
}
union all
{
sql2
}
"
)
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union all
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union all
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
elif
order_union_type
:
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
checkCols
(
1
)
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union all
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
checkCols
(
1
)
elif
rev_order_type
:
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union all
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
else
:
tdSql
.
error
(
f
"
{
sql
1
}
union
{
sql2
}
"
)
tdSql
.
error
(
f
"
{
sql
s
[
i
]
}
union
{
sqls
[
j
+
i
]
}
"
)
def
__test_error
(
self
):
tdSql
.
error
(
"show tables union show tables"
)
tdSql
.
error
(
"create table errtb1 union all create table errtb2"
)
tdSql
.
error
(
"drop table ct1 union all drop table ct3"
)
...
...
tests/system-test/2-query/union2.py
0 → 100644
浏览文件 @
bd5d21b2
import
datetime
from
util.log
import
*
from
util.sql
import
*
from
util.cases
import
*
from
util.dnodes
import
*
PRIMARY_COL
=
"ts"
INT_COL
=
"c1"
BINT_COL
=
"c2"
SINT_COL
=
"c3"
TINT_COL
=
"c4"
FLOAT_COL
=
"c5"
DOUBLE_COL
=
"c6"
BOOL_COL
=
"c7"
BINARY_COL
=
"c8"
NCHAR_COL
=
"c9"
TS_COL
=
"c10"
NUM_COL
=
[
INT_COL
,
BINT_COL
,
SINT_COL
,
TINT_COL
,
FLOAT_COL
,
DOUBLE_COL
,
]
CHAR_COL
=
[
BINARY_COL
,
NCHAR_COL
,
]
BOOLEAN_COL
=
[
BOOL_COL
,
]
TS_TYPE_COL
=
[
TS_COL
,
]
class
TDTestCase
:
def
init
(
self
,
conn
,
logSql
):
tdLog
.
debug
(
f
"start to excute
{
__file__
}
"
)
tdSql
.
init
(
conn
.
cursor
())
def
__query_condition
(
self
,
tbname
):
query_condition
=
[]
for
char_col
in
CHAR_COL
:
query_condition
.
extend
(
(
f
"
{
tbname
}
.
{
char_col
}
"
,
f
"upper(
{
tbname
}
.
{
char_col
}
)"
,
f
"char_length(
{
tbname
}
.
{
char_col
}
)"
,
f
"concat(
{
tbname
}
.
{
char_col
}
,
{
tbname
}
.
{
char_col
}
)"
,
)
)
for
num_col
in
NUM_COL
:
query_condition
.
extend
(
(
f
"
{
tbname
}
.
{
num_col
}
"
,
f
"ceil(
{
tbname
}
.
{
num_col
}
)"
,
f
"abs(
{
tbname
}
.
{
num_col
}
)"
,
f
"acos(
{
tbname
}
.
{
num_col
}
)"
,
f
"max(
{
tbname
}
.
{
num_col
}
)"
,
)
)
query_condition
.
extend
(
(
# 1010,
)
)
return
query_condition
def
__join_condition
(
self
,
tb_list
,
filter
=
PRIMARY_COL
,
INNER
=
False
):
table_reference
=
tb_list
[
0
]
join_condition
=
table_reference
join
=
"inner join"
if
INNER
else
"join"
for
i
in
range
(
len
(
tb_list
[
1
:])):
join_condition
+=
f
"
{
join
}
{
tb_list
[
i
+
1
]
}
on
{
table_reference
}
.
{
filter
}
=
{
tb_list
[
i
+
1
]
}
.
{
filter
}
"
return
join_condition
def
__where_condition
(
self
,
col
=
None
,
tbname
=
None
,
query_conditon
=
None
):
if
query_conditon
and
isinstance
(
query_conditon
,
str
):
if
query_conditon
.
startswith
(
"count"
):
query_conditon
=
query_conditon
[
6
:
-
1
]
elif
query_conditon
.
startswith
(
"max"
):
query_conditon
=
query_conditon
[
4
:
-
1
]
elif
query_conditon
.
startswith
(
"sum"
):
query_conditon
=
query_conditon
[
4
:
-
1
]
elif
query_conditon
.
startswith
(
"min"
):
query_conditon
=
query_conditon
[
4
:
-
1
]
if
query_conditon
:
return
f
" where
{
query_conditon
}
is not null"
if
col
in
NUM_COL
:
return
f
" where abs(
{
tbname
}
.
{
col
}
) >= 0"
if
col
in
CHAR_COL
:
return
f
" where lower(
{
tbname
}
.
{
col
}
) like 'bina%' or lower(
{
tbname
}
.
{
col
}
) like '_cha%' "
if
col
in
BOOLEAN_COL
:
return
f
" where
{
tbname
}
.
{
col
}
in (false, true) "
if
col
in
TS_TYPE_COL
or
col
in
PRIMARY_COL
:
return
f
" where cast(
{
tbname
}
.
{
col
}
as binary(16) ) is not null "
return
""
def
__group_condition
(
self
,
col
,
having
=
None
):
if
isinstance
(
col
,
str
):
if
col
.
startswith
(
"count"
):
col
=
col
[
6
:
-
1
]
elif
col
.
startswith
(
"max"
):
col
=
col
[
4
:
-
1
]
elif
col
.
startswith
(
"sum"
):
col
=
col
[
4
:
-
1
]
elif
col
.
startswith
(
"min"
):
col
=
col
[
4
:
-
1
]
return
f
" group by
{
col
}
having
{
having
}
"
if
having
else
f
" group by
{
col
}
"
def
__single_sql
(
self
,
select_clause
,
from_clause
,
where_condition
=
""
,
group_condition
=
""
):
if
isinstance
(
select_clause
,
str
)
and
"on"
not
in
from_clause
and
select_clause
.
split
(
"."
)[
0
]
!=
from_clause
.
split
(
"."
)[
0
]:
return
return
f
"select
{
select_clause
}
from
{
from_clause
}
{
where_condition
}
{
group_condition
}
"
@
property
def
__join_tblist
(
self
):
return
[
[
"ct1"
,
"ct2"
],
[
"ct1"
,
"ct4"
],
[
"ct1"
,
"t1"
],
[
"ct2"
,
"ct4"
],
[
"ct2"
,
"t1"
],
[
"ct4"
,
"t1"
],
# ["ct1", "ct2", "ct4"],
# ["ct1", "ct2", "t1"],
# ["ct1", "ct4", "t1"],
# ["ct2", "ct4", "t1"],
# ["ct1", "ct2", "ct4", "t1"],
]
@
property
def
__tb_liast
(
self
):
return
[
"ct1"
,
"ct2"
,
"ct4"
,
"t1"
,
]
def
sql_list
(
self
):
sqls
=
[]
__join_tblist
=
self
.
__join_tblist
for
join_tblist
in
__join_tblist
:
for
join_tb
in
join_tblist
:
select_claus_list
=
self
.
__query_condition
(
join_tb
)
for
select_claus
in
select_claus_list
:
group_claus
=
self
.
__group_condition
(
col
=
select_claus
)
where_claus
=
self
.
__where_condition
(
query_conditon
=
select_claus
)
having_claus
=
self
.
__group_condition
(
col
=
select_claus
,
having
=
f
"
{
select_claus
}
is not null"
)
sqls
.
extend
(
(
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
,
group_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
,
having_claus
),
self
.
__single_sql
(
select_claus
,
self
.
__join_condition
(
join_tblist
),
where_claus
,
having_claus
),
self
.
__single_sql
(
select_claus
,
self
.
__join_condition
(
join_tblist
,
INNER
=
True
),
where_claus
,
having_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
having_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
group_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
),
)
)
__no_join_tblist
=
self
.
__tb_liast
for
tb
in
__no_join_tblist
:
select_claus_list
=
self
.
__query_condition
(
tb
)
for
select_claus
in
select_claus_list
:
group_claus
=
self
.
__group_condition
(
col
=
select_claus
)
where_claus
=
self
.
__where_condition
(
query_conditon
=
select_claus
)
having_claus
=
self
.
__group_condition
(
col
=
select_claus
,
having
=
f
"
{
select_claus
}
is not null"
)
sqls
.
extend
(
(
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
,
group_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
,
having_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
group_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
having_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
),
)
)
# return filter(None, sqls)
return
list
(
filter
(
None
,
sqls
))
def
__get_type
(
self
,
col
):
if
tdSql
.
cursor
.
istype
(
col
,
"BOOL"
):
return
"BOOL"
if
tdSql
.
cursor
.
istype
(
col
,
"INT"
):
return
"INT"
if
tdSql
.
cursor
.
istype
(
col
,
"BIGINT"
):
return
"BIGINT"
if
tdSql
.
cursor
.
istype
(
col
,
"TINYINT"
):
return
"TINYINT"
if
tdSql
.
cursor
.
istype
(
col
,
"SMALLINT"
):
return
"SMALLINT"
if
tdSql
.
cursor
.
istype
(
col
,
"FLOAT"
):
return
"FLOAT"
if
tdSql
.
cursor
.
istype
(
col
,
"DOUBLE"
):
return
"DOUBLE"
if
tdSql
.
cursor
.
istype
(
col
,
"BINARY"
):
return
"BINARY"
if
tdSql
.
cursor
.
istype
(
col
,
"NCHAR"
):
return
"NCHAR"
if
tdSql
.
cursor
.
istype
(
col
,
"TIMESTAMP"
):
return
"TIMESTAMP"
if
tdSql
.
cursor
.
istype
(
col
,
"JSON"
):
return
"JSON"
if
tdSql
.
cursor
.
istype
(
col
,
"TINYINT UNSIGNED"
):
return
"TINYINT UNSIGNED"
if
tdSql
.
cursor
.
istype
(
col
,
"SMALLINT UNSIGNED"
):
return
"SMALLINT UNSIGNED"
if
tdSql
.
cursor
.
istype
(
col
,
"INT UNSIGNED"
):
return
"INT UNSIGNED"
if
tdSql
.
cursor
.
istype
(
col
,
"BIGINT UNSIGNED"
):
return
"BIGINT UNSIGNED"
def
union_check
(
self
):
sqls
=
self
.
sql_list
()
for
i
in
range
(
len
(
sqls
)):
tdSql
.
query
(
sqls
[
i
])
res1_type
=
self
.
__get_type
(
0
)
for
j
in
range
(
len
(
sqls
[
i
:])):
tdSql
.
query
(
sqls
[
j
+
i
])
order_union_type
=
False
rev_order_type
=
False
all_union_type
=
False
res2_type
=
self
.
__get_type
(
0
)
if
res2_type
==
res1_type
:
all_union_type
=
True
elif
res1_type
in
(
"BIGINT"
,
"NCHAR"
)
and
res2_type
in
(
"BIGINT"
,
"NCHAR"
):
all_union_type
=
True
elif
res1_type
in
(
"BIGINT"
,
"NCHAR"
):
order_union_type
=
True
elif
res2_type
in
(
"BIGINT"
,
"NCHAR"
):
rev_order_type
=
True
elif
res1_type
==
"TIMESAMP"
and
res2_type
not
in
(
"BINARY"
,
"NCHAR"
):
order_union_type
=
True
elif
res2_type
==
"TIMESAMP"
and
res1_type
not
in
(
"BINARY"
,
"NCHAR"
):
rev_order_type
=
True
elif
res1_type
==
"BINARY"
and
res2_type
!=
"NCHAR"
:
order_union_type
=
True
elif
res2_type
==
"BINARY"
and
res1_type
!=
"NCHAR"
:
rev_order_type
=
True
if
all_union_type
:
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union all
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union all
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
elif
order_union_type
:
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
checkCols
(
1
)
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union all
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
checkCols
(
1
)
elif
rev_order_type
:
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union all
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
else
:
tdSql
.
error
(
f
"
{
sqls
[
i
]
}
union
{
sqls
[
j
+
i
]
}
"
)
def
__test_error
(
self
):
tdSql
.
error
(
"show tables union show tables"
)
tdSql
.
error
(
"create table errtb1 union all create table errtb2"
)
tdSql
.
error
(
"drop table ct1 union all drop table ct3"
)
tdSql
.
error
(
"select c1 from ct1 union all drop table ct3"
)
tdSql
.
error
(
"select c1 from ct1 union all '' "
)
tdSql
.
error
(
" '' union all select c1 from ct1 "
)
tdSql
.
error
(
"select c1 from ct1 union select c1 from ct2 union select c1 from ct4 "
)
def
all_test
(
self
):
self
.
__test_error
()
self
.
union_check
()
def
__create_tb
(
self
):
tdLog
.
printNoPrefix
(
"==========step1:create table"
)
create_stb_sql
=
f
'''create table stb1(
ts timestamp,
{
INT_COL
}
int,
{
BINT_COL
}
bigint,
{
SINT_COL
}
smallint,
{
TINT_COL
}
tinyint,
{
FLOAT_COL
}
float,
{
DOUBLE_COL
}
double,
{
BOOL_COL
}
bool,
{
BINARY_COL
}
binary(16),
{
NCHAR_COL
}
nchar(32),
{
TS_COL
}
timestamp
) tags (t1 int)
'''
create_ntb_sql
=
f
'''create table t1(
ts timestamp,
{
INT_COL
}
int,
{
BINT_COL
}
bigint,
{
SINT_COL
}
smallint,
{
TINT_COL
}
tinyint,
{
FLOAT_COL
}
float,
{
DOUBLE_COL
}
double,
{
BOOL_COL
}
bool,
{
BINARY_COL
}
binary(16),
{
NCHAR_COL
}
nchar(32),
{
TS_COL
}
timestamp
)
'''
tdSql
.
execute
(
create_stb_sql
)
tdSql
.
execute
(
create_ntb_sql
)
for
i
in
range
(
4
):
tdSql
.
execute
(
f
'create table ct
{
i
+
1
}
using stb1 tags (
{
i
+
1
}
)'
)
{
i
%
32767
},
{
i
%
127
},
{
i
*
1.11111
},
{
i
*
1000.1111
},
{
i
%
2
}
def
__insert_data
(
self
,
rows
):
now_time
=
int
(
datetime
.
datetime
.
timestamp
(
datetime
.
datetime
.
now
())
*
1000
)
for
i
in
range
(
rows
):
tdSql
.
execute
(
f
"insert into ct1 values (
{
now_time
-
i
*
1000
}
,
{
i
}
,
{
11111
*
i
}
,
{
111
*
i
%
32767
}
,
{
11
*
i
%
127
}
,
{
1.11
*
i
}
,
{
1100.0011
*
i
}
,
{
i
%
2
}
, 'binary
{
i
}
', 'nchar_测试_
{
i
}
',
{
now_time
+
1
*
i
}
)"
)
tdSql
.
execute
(
f
"insert into ct4 values (
{
now_time
-
i
*
7776000000
}
,
{
i
}
,
{
11111
*
i
}
,
{
111
*
i
%
32767
}
,
{
11
*
i
%
127
}
,
{
1.11
*
i
}
,
{
1100.0011
*
i
}
,
{
i
%
2
}
, 'binary
{
i
}
', 'nchar_测试_
{
i
}
',
{
now_time
+
1
*
i
}
)"
)
tdSql
.
execute
(
f
"insert into ct2 values (
{
now_time
-
i
*
7776000000
}
,
{
-
i
}
,
{
-
11111
*
i
}
,
{
-
111
*
i
%
32767
}
,
{
-
11
*
i
%
127
}
,
{
-
1.11
*
i
}
,
{
-
1100.0011
*
i
}
,
{
i
%
2
}
, 'binary
{
i
}
', 'nchar_测试_
{
i
}
',
{
now_time
+
1
*
i
}
)"
)
tdSql
.
execute
(
f
'''insert into ct1 values
(
{
now_time
-
rows
*
5
}
, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0',
{
now_time
+
8
}
)
(
{
now_time
+
10000
}
,
{
rows
}
, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9',
{
now_time
+
9
}
)
'''
)
tdSql
.
execute
(
f
'''insert into ct4 values
(
{
now_time
-
rows
*
7776000000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
-
rows
*
3888000000
+
10800000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
7776000000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
5184000000
}
,
{
pow
(
2
,
31
)
-
pow
(
2
,
15
)
}
,
{
pow
(
2
,
63
)
-
pow
(
2
,
30
)
}
, 32767, 127,
{
3.3
*
pow
(
10
,
38
)
}
,
{
1.3
*
pow
(
10
,
308
)
}
,
{
rows
%
2
}
, "binary_limit-1", "nchar_测试_limit-1",
{
now_time
-
86400000
}
)
(
{
now_time
+
2592000000
}
,
{
pow
(
2
,
31
)
-
pow
(
2
,
16
)
}
,
{
pow
(
2
,
63
)
-
pow
(
2
,
31
)
}
, 32766, 126,
{
3.2
*
pow
(
10
,
38
)
}
,
{
1.2
*
pow
(
10
,
308
)
}
,
{
(
rows
-
1
)
%
2
}
, "binary_limit-2", "nchar_测试_limit-2",
{
now_time
-
172800000
}
)
'''
)
tdSql
.
execute
(
f
'''insert into ct2 values
(
{
now_time
-
rows
*
7776000000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
-
rows
*
3888000000
+
10800000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
7776000000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
5184000000
}
,
{
-
1
*
pow
(
2
,
31
)
+
pow
(
2
,
15
)
}
,
{
-
1
*
pow
(
2
,
63
)
+
pow
(
2
,
30
)
}
, -32766, -126,
{
-
1
*
3.2
*
pow
(
10
,
38
)
}
,
{
-
1.2
*
pow
(
10
,
308
)
}
,
{
rows
%
2
}
, "binary_limit-1", "nchar_测试_limit-1",
{
now_time
-
86400000
}
)
(
{
now_time
+
2592000000
}
,
{
-
1
*
pow
(
2
,
31
)
+
pow
(
2
,
16
)
}
,
{
-
1
*
pow
(
2
,
63
)
+
pow
(
2
,
31
)
}
, -32767, -127,
{
-
3.3
*
pow
(
10
,
38
)
}
,
{
-
1.3
*
pow
(
10
,
308
)
}
,
{
(
rows
-
1
)
%
2
}
, "binary_limit-2", "nchar_测试_limit-2",
{
now_time
-
172800000
}
)
'''
)
for
i
in
range
(
rows
):
insert_data
=
f
'''insert into t1 values
(
{
now_time
-
i
*
3600000
}
,
{
i
}
,
{
i
*
11111
}
,
{
i
%
32767
}
,
{
i
%
127
}
,
{
i
*
1.11111
}
,
{
i
*
1000.1111
}
,
{
i
%
2
}
,
"binary_
{
i
}
", "nchar_测试_
{
i
}
",
{
now_time
-
1000
*
i
}
)
'''
tdSql
.
execute
(
insert_data
)
tdSql
.
execute
(
f
'''insert into t1 values
(
{
now_time
+
10800000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
-
((
rows
//
2
)
*
60
+
30
)
*
60000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
-
rows
*
3600000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
7200000
}
,
{
pow
(
2
,
31
)
-
pow
(
2
,
15
)
}
,
{
pow
(
2
,
63
)
-
pow
(
2
,
30
)
}
, 32767, 127,
{
3.3
*
pow
(
10
,
38
)
}
,
{
1.3
*
pow
(
10
,
308
)
}
,
{
rows
%
2
}
,
"binary_limit-1", "nchar_测试_limit-1",
{
now_time
-
86400000
}
)
(
{
now_time
+
3600000
}
,
{
pow
(
2
,
31
)
-
pow
(
2
,
16
)
}
,
{
pow
(
2
,
63
)
-
pow
(
2
,
31
)
}
, 32766, 126,
{
3.2
*
pow
(
10
,
38
)
}
,
{
1.2
*
pow
(
10
,
308
)
}
,
{
(
rows
-
1
)
%
2
}
,
"binary_limit-2", "nchar_测试_limit-2",
{
now_time
-
172800000
}
)
'''
)
def
run
(
self
):
tdSql
.
prepare
()
tdLog
.
printNoPrefix
(
"==========step1:create table"
)
self
.
__create_tb
()
tdLog
.
printNoPrefix
(
"==========step2:insert data"
)
self
.
rows
=
10
self
.
__insert_data
(
self
.
rows
)
tdLog
.
printNoPrefix
(
"==========step3:all check"
)
self
.
all_test
()
tdDnodes
.
stop
(
1
)
tdDnodes
.
start
(
1
)
tdSql
.
execute
(
"use db"
)
tdLog
.
printNoPrefix
(
"==========step4:after wal, all check again "
)
self
.
all_test
()
def
stop
(
self
):
tdSql
.
close
()
tdLog
.
success
(
f
"
{
__file__
}
successfully executed"
)
tdCases
.
addLinux
(
__file__
,
TDTestCase
())
tdCases
.
addWindows
(
__file__
,
TDTestCase
())
tests/system-test/2-query/union3.py
0 → 100644
浏览文件 @
bd5d21b2
import
datetime
from
util.log
import
*
from
util.sql
import
*
from
util.cases
import
*
from
util.dnodes
import
*
PRIMARY_COL
=
"ts"
INT_COL
=
"c1"
BINT_COL
=
"c2"
SINT_COL
=
"c3"
TINT_COL
=
"c4"
FLOAT_COL
=
"c5"
DOUBLE_COL
=
"c6"
BOOL_COL
=
"c7"
BINARY_COL
=
"c8"
NCHAR_COL
=
"c9"
TS_COL
=
"c10"
NUM_COL
=
[
INT_COL
,
BINT_COL
,
SINT_COL
,
TINT_COL
,
FLOAT_COL
,
DOUBLE_COL
,
]
CHAR_COL
=
[
BINARY_COL
,
NCHAR_COL
,
]
BOOLEAN_COL
=
[
BOOL_COL
,
]
TS_TYPE_COL
=
[
TS_COL
,
]
class
TDTestCase
:
def
init
(
self
,
conn
,
logSql
):
tdLog
.
debug
(
f
"start to excute
{
__file__
}
"
)
tdSql
.
init
(
conn
.
cursor
())
def
__query_condition
(
self
,
tbname
):
query_condition
=
[]
for
char_col
in
CHAR_COL
:
query_condition
.
extend
(
(
f
"concat_ws( '_',
{
tbname
}
.
{
char_col
}
,
{
tbname
}
.
{
char_col
}
)"
,
f
"length(
{
tbname
}
.
{
char_col
}
)"
,
f
"lower(
{
tbname
}
.
{
char_col
}
)"
,
f
"ltrim(
{
tbname
}
.
{
char_col
}
)"
,
)
)
for
num_col
in
NUM_COL
:
query_condition
.
extend
(
(
f
"asin(
{
tbname
}
.
{
num_col
}
)"
,
f
"atan(
{
tbname
}
.
{
num_col
}
)"
,
f
"cos(
{
tbname
}
.
{
num_col
}
)"
,
f
"sum(
{
tbname
}
.
{
num_col
}
)"
,
)
)
query_condition
.
extend
(
f
"
{
tbname
}
.
{
num_col
}
+
{
tbname
}
.
{
num_col_2
}
"
for
num_col_2
in
NUM_COL
)
query_condition
.
extend
(
(
''' "test1234!@#$%^&*():'><?/.,][}{" '''
,
# 1010,
)
)
return
query_condition
def
__join_condition
(
self
,
tb_list
,
filter
=
PRIMARY_COL
,
INNER
=
False
):
table_reference
=
tb_list
[
0
]
join_condition
=
table_reference
join
=
"inner join"
if
INNER
else
"join"
for
i
in
range
(
len
(
tb_list
[
1
:])):
join_condition
+=
f
"
{
join
}
{
tb_list
[
i
+
1
]
}
on
{
table_reference
}
.
{
filter
}
=
{
tb_list
[
i
+
1
]
}
.
{
filter
}
"
return
join_condition
def
__where_condition
(
self
,
col
=
None
,
tbname
=
None
,
query_conditon
=
None
):
if
query_conditon
and
isinstance
(
query_conditon
,
str
):
if
query_conditon
.
startswith
(
"count"
):
query_conditon
=
query_conditon
[
6
:
-
1
]
elif
query_conditon
.
startswith
(
"max"
):
query_conditon
=
query_conditon
[
4
:
-
1
]
elif
query_conditon
.
startswith
(
"sum"
):
query_conditon
=
query_conditon
[
4
:
-
1
]
elif
query_conditon
.
startswith
(
"min"
):
query_conditon
=
query_conditon
[
4
:
-
1
]
if
query_conditon
:
return
f
" where
{
query_conditon
}
is not null"
if
col
in
NUM_COL
:
return
f
" where abs(
{
tbname
}
.
{
col
}
) >= 0"
if
col
in
CHAR_COL
:
return
f
" where lower(
{
tbname
}
.
{
col
}
) like 'bina%' or lower(
{
tbname
}
.
{
col
}
) like '_cha%' "
if
col
in
BOOLEAN_COL
:
return
f
" where
{
tbname
}
.
{
col
}
in (false, true) "
if
col
in
TS_TYPE_COL
or
col
in
PRIMARY_COL
:
return
f
" where cast(
{
tbname
}
.
{
col
}
as binary(16) ) is not null "
return
""
def
__group_condition
(
self
,
col
,
having
=
None
):
if
isinstance
(
col
,
str
):
if
col
.
startswith
(
"count"
):
col
=
col
[
6
:
-
1
]
elif
col
.
startswith
(
"max"
):
col
=
col
[
4
:
-
1
]
elif
col
.
startswith
(
"sum"
):
col
=
col
[
4
:
-
1
]
elif
col
.
startswith
(
"min"
):
col
=
col
[
4
:
-
1
]
return
f
" group by
{
col
}
having
{
having
}
"
if
having
else
f
" group by
{
col
}
"
def
__single_sql
(
self
,
select_clause
,
from_clause
,
where_condition
=
""
,
group_condition
=
""
):
if
isinstance
(
select_clause
,
str
)
and
"on"
not
in
from_clause
and
select_clause
.
split
(
"."
)[
0
]
!=
from_clause
.
split
(
"."
)[
0
]:
return
return
f
"select
{
select_clause
}
from
{
from_clause
}
{
where_condition
}
{
group_condition
}
"
@
property
def
__join_tblist
(
self
):
return
[
[
"ct1"
,
"ct2"
],
[
"ct1"
,
"ct4"
],
[
"ct1"
,
"t1"
],
[
"ct2"
,
"ct4"
],
[
"ct2"
,
"t1"
],
[
"ct4"
,
"t1"
],
# ["ct1", "ct2", "ct4"],
# ["ct1", "ct2", "t1"],
# ["ct1", "ct4", "t1"],
# ["ct2", "ct4", "t1"],
# ["ct1", "ct2", "ct4", "t1"],
]
@
property
def
__tb_liast
(
self
):
return
[
"ct1"
,
"ct2"
,
"ct4"
,
"t1"
,
]
def
sql_list
(
self
):
sqls
=
[]
__join_tblist
=
self
.
__join_tblist
for
join_tblist
in
__join_tblist
:
for
join_tb
in
join_tblist
:
select_claus_list
=
self
.
__query_condition
(
join_tb
)
for
select_claus
in
select_claus_list
:
group_claus
=
self
.
__group_condition
(
col
=
select_claus
)
where_claus
=
self
.
__where_condition
(
query_conditon
=
select_claus
)
having_claus
=
self
.
__group_condition
(
col
=
select_claus
,
having
=
f
"
{
select_claus
}
is not null"
)
sqls
.
extend
(
(
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
,
group_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
,
having_claus
),
self
.
__single_sql
(
select_claus
,
self
.
__join_condition
(
join_tblist
),
where_claus
,
having_claus
),
self
.
__single_sql
(
select_claus
,
self
.
__join_condition
(
join_tblist
,
INNER
=
True
),
where_claus
,
having_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
having_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
group_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
),
)
)
__no_join_tblist
=
self
.
__tb_liast
for
tb
in
__no_join_tblist
:
select_claus_list
=
self
.
__query_condition
(
tb
)
for
select_claus
in
select_claus_list
:
group_claus
=
self
.
__group_condition
(
col
=
select_claus
)
where_claus
=
self
.
__where_condition
(
query_conditon
=
select_claus
)
having_claus
=
self
.
__group_condition
(
col
=
select_claus
,
having
=
f
"
{
select_claus
}
is not null"
)
sqls
.
extend
(
(
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
,
group_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
,
having_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
group_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
having_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
),
)
)
# return filter(None, sqls)
return
list
(
filter
(
None
,
sqls
))
def
__get_type
(
self
,
col
):
if
tdSql
.
cursor
.
istype
(
col
,
"BOOL"
):
return
"BOOL"
if
tdSql
.
cursor
.
istype
(
col
,
"INT"
):
return
"INT"
if
tdSql
.
cursor
.
istype
(
col
,
"BIGINT"
):
return
"BIGINT"
if
tdSql
.
cursor
.
istype
(
col
,
"TINYINT"
):
return
"TINYINT"
if
tdSql
.
cursor
.
istype
(
col
,
"SMALLINT"
):
return
"SMALLINT"
if
tdSql
.
cursor
.
istype
(
col
,
"FLOAT"
):
return
"FLOAT"
if
tdSql
.
cursor
.
istype
(
col
,
"DOUBLE"
):
return
"DOUBLE"
if
tdSql
.
cursor
.
istype
(
col
,
"BINARY"
):
return
"BINARY"
if
tdSql
.
cursor
.
istype
(
col
,
"NCHAR"
):
return
"NCHAR"
if
tdSql
.
cursor
.
istype
(
col
,
"TIMESTAMP"
):
return
"TIMESTAMP"
if
tdSql
.
cursor
.
istype
(
col
,
"JSON"
):
return
"JSON"
if
tdSql
.
cursor
.
istype
(
col
,
"TINYINT UNSIGNED"
):
return
"TINYINT UNSIGNED"
if
tdSql
.
cursor
.
istype
(
col
,
"SMALLINT UNSIGNED"
):
return
"SMALLINT UNSIGNED"
if
tdSql
.
cursor
.
istype
(
col
,
"INT UNSIGNED"
):
return
"INT UNSIGNED"
if
tdSql
.
cursor
.
istype
(
col
,
"BIGINT UNSIGNED"
):
return
"BIGINT UNSIGNED"
def
union_check
(
self
):
sqls
=
self
.
sql_list
()
for
i
in
range
(
len
(
sqls
)):
tdSql
.
query
(
sqls
[
i
])
res1_type
=
self
.
__get_type
(
0
)
for
j
in
range
(
len
(
sqls
[
i
:])):
tdSql
.
query
(
sqls
[
j
+
i
])
order_union_type
=
False
rev_order_type
=
False
all_union_type
=
False
res2_type
=
self
.
__get_type
(
0
)
if
res2_type
==
res1_type
:
all_union_type
=
True
elif
res1_type
in
(
"BIGINT"
,
"NCHAR"
)
and
res2_type
in
(
"BIGINT"
,
"NCHAR"
):
all_union_type
=
True
elif
res1_type
in
(
"BIGINT"
,
"NCHAR"
):
order_union_type
=
True
elif
res2_type
in
(
"BIGINT"
,
"NCHAR"
):
rev_order_type
=
True
elif
res1_type
==
"TIMESAMP"
and
res2_type
not
in
(
"BINARY"
,
"NCHAR"
):
order_union_type
=
True
elif
res2_type
==
"TIMESAMP"
and
res1_type
not
in
(
"BINARY"
,
"NCHAR"
):
rev_order_type
=
True
elif
res1_type
==
"BINARY"
and
res2_type
!=
"NCHAR"
:
order_union_type
=
True
elif
res2_type
==
"BINARY"
and
res1_type
!=
"NCHAR"
:
rev_order_type
=
True
if
all_union_type
:
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union all
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union all
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
elif
order_union_type
:
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
checkCols
(
1
)
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union all
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
checkCols
(
1
)
elif
rev_order_type
:
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union all
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
else
:
tdSql
.
error
(
f
"
{
sqls
[
i
]
}
union
{
sqls
[
j
+
i
]
}
"
)
def
__test_error
(
self
):
tdSql
.
error
(
"show tables union show tables"
)
tdSql
.
error
(
"create table errtb1 union all create table errtb2"
)
tdSql
.
error
(
"drop table ct1 union all drop table ct3"
)
tdSql
.
error
(
"select c1 from ct1 union all drop table ct3"
)
tdSql
.
error
(
"select c1 from ct1 union all '' "
)
tdSql
.
error
(
" '' union all select c1 from ct1 "
)
tdSql
.
error
(
"select c1 from ct1 union select c1 from ct2 union select c1 from ct4 "
)
def
all_test
(
self
):
self
.
__test_error
()
self
.
union_check
()
def
__create_tb
(
self
):
tdLog
.
printNoPrefix
(
"==========step1:create table"
)
create_stb_sql
=
f
'''create table stb1(
ts timestamp,
{
INT_COL
}
int,
{
BINT_COL
}
bigint,
{
SINT_COL
}
smallint,
{
TINT_COL
}
tinyint,
{
FLOAT_COL
}
float,
{
DOUBLE_COL
}
double,
{
BOOL_COL
}
bool,
{
BINARY_COL
}
binary(16),
{
NCHAR_COL
}
nchar(32),
{
TS_COL
}
timestamp
) tags (t1 int)
'''
create_ntb_sql
=
f
'''create table t1(
ts timestamp,
{
INT_COL
}
int,
{
BINT_COL
}
bigint,
{
SINT_COL
}
smallint,
{
TINT_COL
}
tinyint,
{
FLOAT_COL
}
float,
{
DOUBLE_COL
}
double,
{
BOOL_COL
}
bool,
{
BINARY_COL
}
binary(16),
{
NCHAR_COL
}
nchar(32),
{
TS_COL
}
timestamp
)
'''
tdSql
.
execute
(
create_stb_sql
)
tdSql
.
execute
(
create_ntb_sql
)
for
i
in
range
(
4
):
tdSql
.
execute
(
f
'create table ct
{
i
+
1
}
using stb1 tags (
{
i
+
1
}
)'
)
{
i
%
32767
},
{
i
%
127
},
{
i
*
1.11111
},
{
i
*
1000.1111
},
{
i
%
2
}
def
__insert_data
(
self
,
rows
):
now_time
=
int
(
datetime
.
datetime
.
timestamp
(
datetime
.
datetime
.
now
())
*
1000
)
for
i
in
range
(
rows
):
tdSql
.
execute
(
f
"insert into ct1 values (
{
now_time
-
i
*
1000
}
,
{
i
}
,
{
11111
*
i
}
,
{
111
*
i
%
32767
}
,
{
11
*
i
%
127
}
,
{
1.11
*
i
}
,
{
1100.0011
*
i
}
,
{
i
%
2
}
, 'binary
{
i
}
', 'nchar_测试_
{
i
}
',
{
now_time
+
1
*
i
}
)"
)
tdSql
.
execute
(
f
"insert into ct4 values (
{
now_time
-
i
*
7776000000
}
,
{
i
}
,
{
11111
*
i
}
,
{
111
*
i
%
32767
}
,
{
11
*
i
%
127
}
,
{
1.11
*
i
}
,
{
1100.0011
*
i
}
,
{
i
%
2
}
, 'binary
{
i
}
', 'nchar_测试_
{
i
}
',
{
now_time
+
1
*
i
}
)"
)
tdSql
.
execute
(
f
"insert into ct2 values (
{
now_time
-
i
*
7776000000
}
,
{
-
i
}
,
{
-
11111
*
i
}
,
{
-
111
*
i
%
32767
}
,
{
-
11
*
i
%
127
}
,
{
-
1.11
*
i
}
,
{
-
1100.0011
*
i
}
,
{
i
%
2
}
, 'binary
{
i
}
', 'nchar_测试_
{
i
}
',
{
now_time
+
1
*
i
}
)"
)
tdSql
.
execute
(
f
'''insert into ct1 values
(
{
now_time
-
rows
*
5
}
, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0',
{
now_time
+
8
}
)
(
{
now_time
+
10000
}
,
{
rows
}
, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9',
{
now_time
+
9
}
)
'''
)
tdSql
.
execute
(
f
'''insert into ct4 values
(
{
now_time
-
rows
*
7776000000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
-
rows
*
3888000000
+
10800000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
7776000000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
5184000000
}
,
{
pow
(
2
,
31
)
-
pow
(
2
,
15
)
}
,
{
pow
(
2
,
63
)
-
pow
(
2
,
30
)
}
, 32767, 127,
{
3.3
*
pow
(
10
,
38
)
}
,
{
1.3
*
pow
(
10
,
308
)
}
,
{
rows
%
2
}
, "binary_limit-1", "nchar_测试_limit-1",
{
now_time
-
86400000
}
)
(
{
now_time
+
2592000000
}
,
{
pow
(
2
,
31
)
-
pow
(
2
,
16
)
}
,
{
pow
(
2
,
63
)
-
pow
(
2
,
31
)
}
, 32766, 126,
{
3.2
*
pow
(
10
,
38
)
}
,
{
1.2
*
pow
(
10
,
308
)
}
,
{
(
rows
-
1
)
%
2
}
, "binary_limit-2", "nchar_测试_limit-2",
{
now_time
-
172800000
}
)
'''
)
tdSql
.
execute
(
f
'''insert into ct2 values
(
{
now_time
-
rows
*
7776000000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
-
rows
*
3888000000
+
10800000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
7776000000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
5184000000
}
,
{
-
1
*
pow
(
2
,
31
)
+
pow
(
2
,
15
)
}
,
{
-
1
*
pow
(
2
,
63
)
+
pow
(
2
,
30
)
}
, -32766, -126,
{
-
1
*
3.2
*
pow
(
10
,
38
)
}
,
{
-
1.2
*
pow
(
10
,
308
)
}
,
{
rows
%
2
}
, "binary_limit-1", "nchar_测试_limit-1",
{
now_time
-
86400000
}
)
(
{
now_time
+
2592000000
}
,
{
-
1
*
pow
(
2
,
31
)
+
pow
(
2
,
16
)
}
,
{
-
1
*
pow
(
2
,
63
)
+
pow
(
2
,
31
)
}
, -32767, -127,
{
-
3.3
*
pow
(
10
,
38
)
}
,
{
-
1.3
*
pow
(
10
,
308
)
}
,
{
(
rows
-
1
)
%
2
}
, "binary_limit-2", "nchar_测试_limit-2",
{
now_time
-
172800000
}
)
'''
)
for
i
in
range
(
rows
):
insert_data
=
f
'''insert into t1 values
(
{
now_time
-
i
*
3600000
}
,
{
i
}
,
{
i
*
11111
}
,
{
i
%
32767
}
,
{
i
%
127
}
,
{
i
*
1.11111
}
,
{
i
*
1000.1111
}
,
{
i
%
2
}
,
"binary_
{
i
}
", "nchar_测试_
{
i
}
",
{
now_time
-
1000
*
i
}
)
'''
tdSql
.
execute
(
insert_data
)
tdSql
.
execute
(
f
'''insert into t1 values
(
{
now_time
+
10800000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
-
((
rows
//
2
)
*
60
+
30
)
*
60000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
-
rows
*
3600000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
7200000
}
,
{
pow
(
2
,
31
)
-
pow
(
2
,
15
)
}
,
{
pow
(
2
,
63
)
-
pow
(
2
,
30
)
}
, 32767, 127,
{
3.3
*
pow
(
10
,
38
)
}
,
{
1.3
*
pow
(
10
,
308
)
}
,
{
rows
%
2
}
,
"binary_limit-1", "nchar_测试_limit-1",
{
now_time
-
86400000
}
)
(
{
now_time
+
3600000
}
,
{
pow
(
2
,
31
)
-
pow
(
2
,
16
)
}
,
{
pow
(
2
,
63
)
-
pow
(
2
,
31
)
}
, 32766, 126,
{
3.2
*
pow
(
10
,
38
)
}
,
{
1.2
*
pow
(
10
,
308
)
}
,
{
(
rows
-
1
)
%
2
}
,
"binary_limit-2", "nchar_测试_limit-2",
{
now_time
-
172800000
}
)
'''
)
def
run
(
self
):
tdSql
.
prepare
()
tdLog
.
printNoPrefix
(
"==========step1:create table"
)
self
.
__create_tb
()
tdLog
.
printNoPrefix
(
"==========step2:insert data"
)
self
.
rows
=
10
self
.
__insert_data
(
self
.
rows
)
tdLog
.
printNoPrefix
(
"==========step3:all check"
)
self
.
all_test
()
tdDnodes
.
stop
(
1
)
tdDnodes
.
start
(
1
)
tdSql
.
execute
(
"use db"
)
tdLog
.
printNoPrefix
(
"==========step4:after wal, all check again "
)
self
.
all_test
()
def
stop
(
self
):
tdSql
.
close
()
tdLog
.
success
(
f
"
{
__file__
}
successfully executed"
)
tdCases
.
addLinux
(
__file__
,
TDTestCase
())
tdCases
.
addWindows
(
__file__
,
TDTestCase
())
tests/system-test/2-query/union4.py
0 → 100644
浏览文件 @
bd5d21b2
import
datetime
from
util.log
import
*
from
util.sql
import
*
from
util.cases
import
*
from
util.dnodes
import
*
PRIMARY_COL
=
"ts"
INT_COL
=
"c1"
BINT_COL
=
"c2"
SINT_COL
=
"c3"
TINT_COL
=
"c4"
FLOAT_COL
=
"c5"
DOUBLE_COL
=
"c6"
BOOL_COL
=
"c7"
BINARY_COL
=
"c8"
NCHAR_COL
=
"c9"
TS_COL
=
"c10"
NUM_COL
=
[
INT_COL
,
BINT_COL
,
SINT_COL
,
TINT_COL
,
FLOAT_COL
,
DOUBLE_COL
,
]
CHAR_COL
=
[
BINARY_COL
,
NCHAR_COL
,
]
BOOLEAN_COL
=
[
BOOL_COL
,
]
TS_TYPE_COL
=
[
TS_COL
,
]
class
TDTestCase
:
def
init
(
self
,
conn
,
logSql
):
tdLog
.
debug
(
f
"start to excute
{
__file__
}
"
)
tdSql
.
init
(
conn
.
cursor
())
def
__query_condition
(
self
,
tbname
):
query_condition
=
[]
for
char_col
in
CHAR_COL
:
query_condition
.
extend
(
(
f
"cast(
{
tbname
}
.
{
char_col
}
as nchar(8) )"
,
)
)
query_condition
.
extend
(
f
"cast(
{
tbname
}
.
{
un_char_col
}
as binary(16) ) "
for
un_char_col
in
NUM_COL
)
query_condition
.
extend
(
f
"cast(
{
tbname
}
.
{
char_col
}
+
{
tbname
}
.
{
char_col_2
}
as binary(32) ) "
for
char_col_2
in
CHAR_COL
)
query_condition
.
extend
(
f
"cast(
{
tbname
}
.
{
char_col
}
+
{
tbname
}
.
{
un_char_col
}
as binary(32) ) "
for
un_char_col
in
NUM_COL
)
for
num_col
in
NUM_COL
:
query_condition
.
extend
(
(
f
"tan(
{
tbname
}
.
{
num_col
}
)"
,
f
"round(
{
tbname
}
.
{
num_col
}
)"
,
f
"count(
{
tbname
}
.
{
num_col
}
)"
,
f
"min(
{
tbname
}
.
{
num_col
}
)"
,
)
)
query_condition
.
extend
(
f
"
{
tbname
}
.
{
num_col
}
+
{
tbname
}
.
{
char_col
}
"
for
char_col
in
CHAR_COL
)
query_condition
.
extend
(
(
''' "test12" '''
,
# 1010,
)
)
return
query_condition
def
__join_condition
(
self
,
tb_list
,
filter
=
PRIMARY_COL
,
INNER
=
False
):
table_reference
=
tb_list
[
0
]
join_condition
=
table_reference
join
=
"inner join"
if
INNER
else
"join"
for
i
in
range
(
len
(
tb_list
[
1
:])):
join_condition
+=
f
"
{
join
}
{
tb_list
[
i
+
1
]
}
on
{
table_reference
}
.
{
filter
}
=
{
tb_list
[
i
+
1
]
}
.
{
filter
}
"
return
join_condition
def
__where_condition
(
self
,
col
=
None
,
tbname
=
None
,
query_conditon
=
None
):
if
query_conditon
and
isinstance
(
query_conditon
,
str
):
if
query_conditon
.
startswith
(
"count"
):
query_conditon
=
query_conditon
[
6
:
-
1
]
elif
query_conditon
.
startswith
(
"max"
):
query_conditon
=
query_conditon
[
4
:
-
1
]
elif
query_conditon
.
startswith
(
"sum"
):
query_conditon
=
query_conditon
[
4
:
-
1
]
elif
query_conditon
.
startswith
(
"min"
):
query_conditon
=
query_conditon
[
4
:
-
1
]
if
query_conditon
:
return
f
" where
{
query_conditon
}
is not null"
if
col
in
NUM_COL
:
return
f
" where abs(
{
tbname
}
.
{
col
}
) >= 0"
if
col
in
CHAR_COL
:
return
f
" where lower(
{
tbname
}
.
{
col
}
) like 'bina%' or lower(
{
tbname
}
.
{
col
}
) like '_cha%' "
if
col
in
BOOLEAN_COL
:
return
f
" where
{
tbname
}
.
{
col
}
in (false, true) "
if
col
in
TS_TYPE_COL
or
col
in
PRIMARY_COL
:
return
f
" where cast(
{
tbname
}
.
{
col
}
as binary(16) ) is not null "
return
""
def
__group_condition
(
self
,
col
,
having
=
None
):
if
isinstance
(
col
,
str
):
if
col
.
startswith
(
"count"
):
col
=
col
[
6
:
-
1
]
elif
col
.
startswith
(
"max"
):
col
=
col
[
4
:
-
1
]
elif
col
.
startswith
(
"sum"
):
col
=
col
[
4
:
-
1
]
elif
col
.
startswith
(
"min"
):
col
=
col
[
4
:
-
1
]
return
f
" group by
{
col
}
having
{
having
}
"
if
having
else
f
" group by
{
col
}
"
def
__single_sql
(
self
,
select_clause
,
from_clause
,
where_condition
=
""
,
group_condition
=
""
):
if
isinstance
(
select_clause
,
str
)
and
"on"
not
in
from_clause
and
select_clause
.
split
(
"."
)[
0
]
!=
from_clause
.
split
(
"."
)[
0
]:
return
return
f
"select
{
select_clause
}
from
{
from_clause
}
{
where_condition
}
{
group_condition
}
"
@
property
def
__join_tblist
(
self
):
return
[
[
"ct1"
,
"ct2"
],
[
"ct1"
,
"ct4"
],
[
"ct1"
,
"t1"
],
[
"ct2"
,
"ct4"
],
[
"ct2"
,
"t1"
],
[
"ct4"
,
"t1"
],
# ["ct1", "ct2", "ct4"],
# ["ct1", "ct2", "t1"],
# ["ct1", "ct4", "t1"],
# ["ct2", "ct4", "t1"],
# ["ct1", "ct2", "ct4", "t1"],
]
@
property
def
__tb_liast
(
self
):
return
[
"ct1"
,
"ct2"
,
"ct4"
,
"t1"
,
]
def
sql_list
(
self
):
sqls
=
[]
__join_tblist
=
self
.
__join_tblist
for
join_tblist
in
__join_tblist
:
for
join_tb
in
join_tblist
:
select_claus_list
=
self
.
__query_condition
(
join_tb
)
for
select_claus
in
select_claus_list
:
group_claus
=
self
.
__group_condition
(
col
=
select_claus
)
where_claus
=
self
.
__where_condition
(
query_conditon
=
select_claus
)
having_claus
=
self
.
__group_condition
(
col
=
select_claus
,
having
=
f
"
{
select_claus
}
is not null"
)
sqls
.
extend
(
(
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
,
group_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
,
having_claus
),
self
.
__single_sql
(
select_claus
,
self
.
__join_condition
(
join_tblist
),
where_claus
,
having_claus
),
self
.
__single_sql
(
select_claus
,
self
.
__join_condition
(
join_tblist
,
INNER
=
True
),
where_claus
,
having_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
having_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
group_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
),
)
)
__no_join_tblist
=
self
.
__tb_liast
for
tb
in
__no_join_tblist
:
select_claus_list
=
self
.
__query_condition
(
tb
)
for
select_claus
in
select_claus_list
:
group_claus
=
self
.
__group_condition
(
col
=
select_claus
)
where_claus
=
self
.
__where_condition
(
query_conditon
=
select_claus
)
having_claus
=
self
.
__group_condition
(
col
=
select_claus
,
having
=
f
"
{
select_claus
}
is not null"
)
sqls
.
extend
(
(
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
,
group_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
,
having_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
where_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
group_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
,
having_claus
),
self
.
__single_sql
(
select_claus
,
join_tb
),
)
)
# return filter(None, sqls)
return
list
(
filter
(
None
,
sqls
))
def
__get_type
(
self
,
col
):
if
tdSql
.
cursor
.
istype
(
col
,
"BOOL"
):
return
"BOOL"
if
tdSql
.
cursor
.
istype
(
col
,
"INT"
):
return
"INT"
if
tdSql
.
cursor
.
istype
(
col
,
"BIGINT"
):
return
"BIGINT"
if
tdSql
.
cursor
.
istype
(
col
,
"TINYINT"
):
return
"TINYINT"
if
tdSql
.
cursor
.
istype
(
col
,
"SMALLINT"
):
return
"SMALLINT"
if
tdSql
.
cursor
.
istype
(
col
,
"FLOAT"
):
return
"FLOAT"
if
tdSql
.
cursor
.
istype
(
col
,
"DOUBLE"
):
return
"DOUBLE"
if
tdSql
.
cursor
.
istype
(
col
,
"BINARY"
):
return
"BINARY"
if
tdSql
.
cursor
.
istype
(
col
,
"NCHAR"
):
return
"NCHAR"
if
tdSql
.
cursor
.
istype
(
col
,
"TIMESTAMP"
):
return
"TIMESTAMP"
if
tdSql
.
cursor
.
istype
(
col
,
"JSON"
):
return
"JSON"
if
tdSql
.
cursor
.
istype
(
col
,
"TINYINT UNSIGNED"
):
return
"TINYINT UNSIGNED"
if
tdSql
.
cursor
.
istype
(
col
,
"SMALLINT UNSIGNED"
):
return
"SMALLINT UNSIGNED"
if
tdSql
.
cursor
.
istype
(
col
,
"INT UNSIGNED"
):
return
"INT UNSIGNED"
if
tdSql
.
cursor
.
istype
(
col
,
"BIGINT UNSIGNED"
):
return
"BIGINT UNSIGNED"
def
union_check
(
self
):
sqls
=
self
.
sql_list
()
for
i
in
range
(
len
(
sqls
)):
tdSql
.
query
(
sqls
[
i
])
res1_type
=
self
.
__get_type
(
0
)
for
j
in
range
(
len
(
sqls
[
i
:])):
tdSql
.
query
(
sqls
[
j
+
i
])
order_union_type
=
False
rev_order_type
=
False
all_union_type
=
False
res2_type
=
self
.
__get_type
(
0
)
if
res2_type
==
res1_type
:
all_union_type
=
True
elif
res1_type
in
(
"BIGINT"
,
"NCHAR"
)
and
res2_type
in
(
"BIGINT"
,
"NCHAR"
):
all_union_type
=
True
elif
res1_type
in
(
"BIGINT"
,
"NCHAR"
):
order_union_type
=
True
elif
res2_type
in
(
"BIGINT"
,
"NCHAR"
):
rev_order_type
=
True
elif
res1_type
==
"TIMESAMP"
and
res2_type
not
in
(
"BINARY"
,
"NCHAR"
):
order_union_type
=
True
elif
res2_type
==
"TIMESAMP"
and
res1_type
not
in
(
"BINARY"
,
"NCHAR"
):
rev_order_type
=
True
elif
res1_type
==
"BINARY"
and
res2_type
!=
"NCHAR"
:
order_union_type
=
True
elif
res2_type
==
"BINARY"
and
res1_type
!=
"NCHAR"
:
rev_order_type
=
True
if
all_union_type
:
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union all
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union all
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
elif
order_union_type
:
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
checkCols
(
1
)
tdSql
.
query
(
f
"
{
sqls
[
i
]
}
union all
{
sqls
[
j
+
i
]
}
"
)
tdSql
.
checkCols
(
1
)
elif
rev_order_type
:
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
tdSql
.
query
(
f
"
{
sqls
[
j
+
i
]
}
union all
{
sqls
[
i
]
}
"
)
tdSql
.
checkCols
(
1
)
else
:
tdSql
.
error
(
f
"
{
sqls
[
i
]
}
union
{
sqls
[
j
+
i
]
}
"
)
def
__test_error
(
self
):
tdSql
.
error
(
"show tables union show tables"
)
tdSql
.
error
(
"create table errtb1 union all create table errtb2"
)
tdSql
.
error
(
"drop table ct1 union all drop table ct3"
)
tdSql
.
error
(
"select c1 from ct1 union all drop table ct3"
)
tdSql
.
error
(
"select c1 from ct1 union all '' "
)
tdSql
.
error
(
" '' union all select c1 from ct1 "
)
tdSql
.
error
(
"select c1 from ct1 union select c1 from ct2 union select c1 from ct4 "
)
def
all_test
(
self
):
self
.
__test_error
()
self
.
union_check
()
def
__create_tb
(
self
):
tdLog
.
printNoPrefix
(
"==========step1:create table"
)
create_stb_sql
=
f
'''create table stb1(
ts timestamp,
{
INT_COL
}
int,
{
BINT_COL
}
bigint,
{
SINT_COL
}
smallint,
{
TINT_COL
}
tinyint,
{
FLOAT_COL
}
float,
{
DOUBLE_COL
}
double,
{
BOOL_COL
}
bool,
{
BINARY_COL
}
binary(16),
{
NCHAR_COL
}
nchar(32),
{
TS_COL
}
timestamp
) tags (t1 int)
'''
create_ntb_sql
=
f
'''create table t1(
ts timestamp,
{
INT_COL
}
int,
{
BINT_COL
}
bigint,
{
SINT_COL
}
smallint,
{
TINT_COL
}
tinyint,
{
FLOAT_COL
}
float,
{
DOUBLE_COL
}
double,
{
BOOL_COL
}
bool,
{
BINARY_COL
}
binary(16),
{
NCHAR_COL
}
nchar(32),
{
TS_COL
}
timestamp
)
'''
tdSql
.
execute
(
create_stb_sql
)
tdSql
.
execute
(
create_ntb_sql
)
for
i
in
range
(
4
):
tdSql
.
execute
(
f
'create table ct
{
i
+
1
}
using stb1 tags (
{
i
+
1
}
)'
)
{
i
%
32767
},
{
i
%
127
},
{
i
*
1.11111
},
{
i
*
1000.1111
},
{
i
%
2
}
def
__insert_data
(
self
,
rows
):
now_time
=
int
(
datetime
.
datetime
.
timestamp
(
datetime
.
datetime
.
now
())
*
1000
)
for
i
in
range
(
rows
):
tdSql
.
execute
(
f
"insert into ct1 values (
{
now_time
-
i
*
1000
}
,
{
i
}
,
{
11111
*
i
}
,
{
111
*
i
%
32767
}
,
{
11
*
i
%
127
}
,
{
1.11
*
i
}
,
{
1100.0011
*
i
}
,
{
i
%
2
}
, 'binary
{
i
}
', 'nchar_测试_
{
i
}
',
{
now_time
+
1
*
i
}
)"
)
tdSql
.
execute
(
f
"insert into ct4 values (
{
now_time
-
i
*
7776000000
}
,
{
i
}
,
{
11111
*
i
}
,
{
111
*
i
%
32767
}
,
{
11
*
i
%
127
}
,
{
1.11
*
i
}
,
{
1100.0011
*
i
}
,
{
i
%
2
}
, 'binary
{
i
}
', 'nchar_测试_
{
i
}
',
{
now_time
+
1
*
i
}
)"
)
tdSql
.
execute
(
f
"insert into ct2 values (
{
now_time
-
i
*
7776000000
}
,
{
-
i
}
,
{
-
11111
*
i
}
,
{
-
111
*
i
%
32767
}
,
{
-
11
*
i
%
127
}
,
{
-
1.11
*
i
}
,
{
-
1100.0011
*
i
}
,
{
i
%
2
}
, 'binary
{
i
}
', 'nchar_测试_
{
i
}
',
{
now_time
+
1
*
i
}
)"
)
tdSql
.
execute
(
f
'''insert into ct1 values
(
{
now_time
-
rows
*
5
}
, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0',
{
now_time
+
8
}
)
(
{
now_time
+
10000
}
,
{
rows
}
, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9',
{
now_time
+
9
}
)
'''
)
tdSql
.
execute
(
f
'''insert into ct4 values
(
{
now_time
-
rows
*
7776000000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
-
rows
*
3888000000
+
10800000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
7776000000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
5184000000
}
,
{
pow
(
2
,
31
)
-
pow
(
2
,
15
)
}
,
{
pow
(
2
,
63
)
-
pow
(
2
,
30
)
}
, 32767, 127,
{
3.3
*
pow
(
10
,
38
)
}
,
{
1.3
*
pow
(
10
,
308
)
}
,
{
rows
%
2
}
, "binary_limit-1", "nchar_测试_limit-1",
{
now_time
-
86400000
}
)
(
{
now_time
+
2592000000
}
,
{
pow
(
2
,
31
)
-
pow
(
2
,
16
)
}
,
{
pow
(
2
,
63
)
-
pow
(
2
,
31
)
}
, 32766, 126,
{
3.2
*
pow
(
10
,
38
)
}
,
{
1.2
*
pow
(
10
,
308
)
}
,
{
(
rows
-
1
)
%
2
}
, "binary_limit-2", "nchar_测试_limit-2",
{
now_time
-
172800000
}
)
'''
)
tdSql
.
execute
(
f
'''insert into ct2 values
(
{
now_time
-
rows
*
7776000000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
-
rows
*
3888000000
+
10800000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
7776000000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
5184000000
}
,
{
-
1
*
pow
(
2
,
31
)
+
pow
(
2
,
15
)
}
,
{
-
1
*
pow
(
2
,
63
)
+
pow
(
2
,
30
)
}
, -32766, -126,
{
-
1
*
3.2
*
pow
(
10
,
38
)
}
,
{
-
1.2
*
pow
(
10
,
308
)
}
,
{
rows
%
2
}
, "binary_limit-1", "nchar_测试_limit-1",
{
now_time
-
86400000
}
)
(
{
now_time
+
2592000000
}
,
{
-
1
*
pow
(
2
,
31
)
+
pow
(
2
,
16
)
}
,
{
-
1
*
pow
(
2
,
63
)
+
pow
(
2
,
31
)
}
, -32767, -127,
{
-
3.3
*
pow
(
10
,
38
)
}
,
{
-
1.3
*
pow
(
10
,
308
)
}
,
{
(
rows
-
1
)
%
2
}
, "binary_limit-2", "nchar_测试_limit-2",
{
now_time
-
172800000
}
)
'''
)
for
i
in
range
(
rows
):
insert_data
=
f
'''insert into t1 values
(
{
now_time
-
i
*
3600000
}
,
{
i
}
,
{
i
*
11111
}
,
{
i
%
32767
}
,
{
i
%
127
}
,
{
i
*
1.11111
}
,
{
i
*
1000.1111
}
,
{
i
%
2
}
,
"binary_
{
i
}
", "nchar_测试_
{
i
}
",
{
now_time
-
1000
*
i
}
)
'''
tdSql
.
execute
(
insert_data
)
tdSql
.
execute
(
f
'''insert into t1 values
(
{
now_time
+
10800000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
-
((
rows
//
2
)
*
60
+
30
)
*
60000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
-
rows
*
3600000
}
, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{
now_time
+
7200000
}
,
{
pow
(
2
,
31
)
-
pow
(
2
,
15
)
}
,
{
pow
(
2
,
63
)
-
pow
(
2
,
30
)
}
, 32767, 127,
{
3.3
*
pow
(
10
,
38
)
}
,
{
1.3
*
pow
(
10
,
308
)
}
,
{
rows
%
2
}
,
"binary_limit-1", "nchar_测试_limit-1",
{
now_time
-
86400000
}
)
(
{
now_time
+
3600000
}
,
{
pow
(
2
,
31
)
-
pow
(
2
,
16
)
}
,
{
pow
(
2
,
63
)
-
pow
(
2
,
31
)
}
, 32766, 126,
{
3.2
*
pow
(
10
,
38
)
}
,
{
1.2
*
pow
(
10
,
308
)
}
,
{
(
rows
-
1
)
%
2
}
,
"binary_limit-2", "nchar_测试_limit-2",
{
now_time
-
172800000
}
)
'''
)
def
run
(
self
):
tdSql
.
prepare
()
tdLog
.
printNoPrefix
(
"==========step1:create table"
)
self
.
__create_tb
()
tdLog
.
printNoPrefix
(
"==========step2:insert data"
)
self
.
rows
=
10
self
.
__insert_data
(
self
.
rows
)
tdLog
.
printNoPrefix
(
"==========step3:all check"
)
self
.
all_test
()
tdDnodes
.
stop
(
1
)
tdDnodes
.
start
(
1
)
tdSql
.
execute
(
"use db"
)
tdLog
.
printNoPrefix
(
"==========step4:after wal, all check again "
)
self
.
all_test
()
def
stop
(
self
):
tdSql
.
close
()
tdLog
.
success
(
f
"
{
__file__
}
successfully executed"
)
tdCases
.
addLinux
(
__file__
,
TDTestCase
())
tdCases
.
addWindows
(
__file__
,
TDTestCase
())
tests/system-test/7-tmq/subscribeDb1.py
0 → 100644
浏览文件 @
bd5d21b2
此差异已折叠。
点击以展开。
tests/system-test/fulltest.sh
浏览文件 @
bd5d21b2
...
...
@@ -22,8 +22,12 @@ python3 ./test.py -f 2-query/upper.py
python3 ./test.py
-f
2-query/lower.py
python3 ./test.py
-f
2-query/join.py
python3 ./test.py
-f
2-query/cast.py
# python3 ./test.py -f 2-query/concat.py # after wal ,crash occured
# python3 ./test.py -f 2-query/concat_ws.py
python3 ./test.py
-f
2-query/concat.py
python3 ./test.py
-f
2-query/concat_ws.py
# python3 ./test.py -f 2-query/union.py
# python3 ./test.py -f 2-query/union2.py
# python3 ./test.py -f 2-query/union3.py
# python3 ./test.py -f 2-query/union4.py
python3 ./test.py
-f
2-query/timezone.py
python3 ./test.py
-f
2-query/Now.py
...
...
@@ -58,4 +62,4 @@ python3 ./test.py -f 2-query/nestedQuery.py
python3 ./test.py
-f
7-tmq/basic5.py
python3 ./test.py
-f
7-tmq/subscribeDb.py
python3 ./test.py
-f
7-tmq/subscribeDb1.py
tests/test/c/tmqSim.c
浏览文件 @
bd5d21b2
...
...
@@ -278,7 +278,7 @@ void build_consumer(SThreadInfo* pInfo) {
//tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName);
tmq_conf_set_
offset
_commit_cb
(
conf
,
tmq_commit_cb_print
,
NULL
);
tmq_conf_set_
auto
_commit_cb
(
conf
,
tmq_commit_cb_print
,
NULL
);
// tmq_conf_set(conf, "group.id", "cgrp1");
for
(
int32_t
i
=
0
;
i
<
pInfo
->
numOfKey
;
i
++
)
{
...
...
@@ -322,8 +322,11 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) {
sprintf
(
sqlStr
,
"insert into %s.consumeresult values (now, %d, %"
PRId64
", %"
PRId64
", %d)"
,
g_stConfInfo
.
cdbName
,
pInfo
->
consumerId
,
pInfo
->
consumeMsgCnt
,
pInfo
->
consumeRowCnt
,
pInfo
->
checkresult
);
taosFprintfFile
(
g_fp
,
"== save result sql: %s
\n
"
,
sqlStr
);
time_t
tTime
=
taosGetTimestampSec
();
struct
tm
tm
=
*
taosLocalTime
(
&
tTime
,
NULL
);
taosFprintfFile
(
g_fp
,
"# save result: %d-%02d-%02d %02d:%02d:%02d, sql: %s
\n
"
,
tm
.
tm_year
+
1900
,
tm
.
tm_mon
+
1
,
tm
.
tm_mday
,
tm
.
tm_hour
,
tm
.
tm_min
,
tm
.
tm_sec
,
sqlStr
);
TAOS_RES
*
pRes
=
taos_query
(
pConn
,
sqlStr
);
if
(
taos_errno
(
pRes
)
!=
0
)
{
pError
(
"error in save consumeinfo, reason:%s
\n
"
,
taos_errstr
(
pRes
));
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录