Skip to content

T
TDengine

taosdata / TDengine
接近 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
T
TDengine
提交 086d6f39 编写于 作者: G Ganlin Zhao
浏览文件
操作

Merge branch '3.0' into feat/groupkey_function

上级 37d1cc28 f0b50add
隐藏空白更改
内联 并排
Showing with 1954 addition and 904 deletion
Jenkinsfile2
浏览文件 @ 086d6f39
......@@ -43,10 +43,8 @@ def pre_test(){
sh '''
cd ${WK}
git reset --hard
git fetch || git fetch
cd ${WKC}
git reset --hard
git fetch || git fetch
'''
script {
if (env.CHANGE_TARGET == 'master') {
......@@ -82,6 +80,7 @@ def pre_test(){
if (env.CHANGE_URL =~ /\/TDengine\//) {
sh '''
cd ${WKC}
git remote prune origin
git pull >/dev/null
git log -5
echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log
......@@ -107,6 +106,7 @@ def pre_test(){
git log -5
echo "tdinternal log merged: `git log -5`" >>${WKDIR}/jenkins.log
cd ${WKC}
git remote prune origin
git pull >/dev/null
git log -5
echo "community log: `git log -5`" >>${WKDIR}/jenkins.log
......@@ -137,53 +137,51 @@ def pre_test_win(){
set
date /t
time /t
rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug || exit 0
rd /s /Q %WIN_INTERNAL_ROOT%\\debug || exit 0
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
cd %WIN_INTERNAL_ROOT%
git reset --hard
git fetch || git fetch
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
cd %WIN_COMMUNITY_ROOT%
git reset --hard
git fetch || git fetch
'''
script {
if (env.CHANGE_TARGET == 'master') {
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
cd %WIN_INTERNAL_ROOT%
git checkout master
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
cd %WIN_COMMUNITY_ROOT%
git checkout master
'''
} else if(env.CHANGE_TARGET == '2.0') {
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
cd %WIN_INTERNAL_ROOT%
git checkout 2.0
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
cd %WIN_COMMUNITY_ROOT%
git checkout 2.0
'''
} else if(env.CHANGE_TARGET == '3.0') {
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
cd %WIN_INTERNAL_ROOT%
git checkout 3.0
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
cd %WIN_COMMUNITY_ROOT%
git checkout 3.0
'''
} else {
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
cd %WIN_INTERNAL_ROOT%
git checkout develop
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
cd %WIN_COMMUNITY_ROOT%
git checkout develop
'''
}
......@@ -191,36 +189,38 @@ def pre_test_win(){
script {
if (env.CHANGE_URL =~ /\/TDengine\//) {
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
cd %WIN_INTERNAL_ROOT%
git pull
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
cd %WIN_COMMUNITY_ROOT%
git remote prune origin
git pull
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
cd %WIN_COMMUNITY_ROOT%
git fetch origin +refs/pull/%CHANGE_ID%/merge
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
cd %WIN_COMMUNITY_ROOT%
git checkout -qf FETCH_HEAD
'''
} else if (env.CHANGE_URL =~ /\/TDinternal\//) {
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
cd %WIN_INTERNAL_ROOT%
git pull
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
cd %WIN_INTERNAL_ROOT%
git fetch origin +refs/pull/%CHANGE_ID%/merge
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
cd %WIN_INTERNAL_ROOT%
git checkout -qf FETCH_HEAD
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
cd %WIN_COMMUNITY_ROOT%
git remote prune origin
git pull
'''
} else {
......@@ -230,27 +230,27 @@ def pre_test_win(){
}
}
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
cd %WIN_INTERNAL_ROOT%
git branch
git log -5
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
cd %WIN_COMMUNITY_ROOT%
git branch
git log -5
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
cd %WIN_COMMUNITY_ROOT%
git submodule update --init --recursive
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python
cd %WIN_CONNECTOR_ROOT%
git branch
git reset --hard
git pull
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python
cd %WIN_CONNECTOR_ROOT%
git log -5
'''
}
......@@ -258,7 +258,7 @@ def pre_test_build_win() {
bat '''
echo "building ..."
time /t
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
cd %WIN_INTERNAL_ROOT%
mkdir debug
cd debug
time /t
......@@ -273,9 +273,9 @@ def pre_test_build_win() {
time /t
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python
cd %WIN_CONNECTOR_ROOT%
python -m pip install .
xcopy /e/y/i/f C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
'''
return 1
}
......@@ -283,7 +283,7 @@ def run_win_ctest() {
bat '''
echo "windows ctest ..."
time /t
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug
cd %WIN_INTERNAL_ROOT%\\debug
ctest -j 1 || exit 7
time /t
'''
......@@ -292,12 +292,12 @@ def run_win_test() {
echo "LINUX NODE: ${linux_node_ip} - ${linux_node_pass}"
bat '''
echo "windows test ..."
cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python
cd %WIN_CONNECTOR_ROOT%
python -m pip install .
xcopy /e/y/i/f C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
ls -l C:\\Windows\\System32\\taos.dll
time /t
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community\\tests\\system-test
cd %WIN_SYSTEM_TEST_ROOT%
echo "node: ''' + linux_node_ip + ''':''' + linux_node_pass + '''"
echo "testing ..."
test-all.bat "{\\\"host\\\":\\\"''' + linux_node_ip + '''\\\",\\\"port\\\":22,\\\"user\\\":\\\"root\\\",\\\"password\\\":\\\"''' + linux_node_pass + '''\\\",\\\"path\\\":\\\"/var/lib/jenkins/workspace/TDinternal\\\"}"
......@@ -319,6 +319,12 @@ pipeline {
parallel {
stage('windows test') {
agent{label " windows10_01 || windows10_02 || windows10_03 || windows10_04 "}
environment{
WIN_INTERNAL_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal"
WIN_COMMUNITY_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal\\community"
WIN_SYSTEM_TEST_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal\\community\\tests\\system-test"
WIN_CONNECTOR_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\taos-connector-python"
}
steps {
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
timeout(time: 55, unit: 'MINUTES'){
......@@ -357,7 +363,7 @@ pipeline {
echo "${linux_node_ip}:${linux_node_pass}"
}
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
timeout(time: 40, unit: 'MINUTES'){
timeout(time: 120, unit: 'MINUTES'){
pre_test()
script {
sh '''
......@@ -368,11 +374,36 @@ pipeline {
rm -f /tmp/cases.task
./collect_cases.sh -e
'''
def extra_param = ""
def log_server_file = "/home/log_server.json"
def timeout_cmd = ""
if (fileExists(log_server_file)) {
def log_server_enabled = sh (
script: 'jq .enabled ' + log_server_file,
returnStdout: true
).trim()
def timeout_param = sh (
script: 'jq .timeout ' + log_server_file,
returnStdout: true
).trim()
if (timeout_param != "null" && timeout_param != "0") {
timeout_cmd = "timeout " + timeout_param
}
if (log_server_enabled == "1") {
def log_server = sh (
script: 'jq .server ' + log_server_file + ' | sed "s/\\\"//g"',
returnStdout: true
).trim()
if (log_server != "null" && log_server != "") {
extra_param = "-w " + log_server
}
}
}
sh '''
cd ${WKC}/tests/parallel_test
export DEFAULT_RETRY_TIME=2
date
timeout 2100 time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480
''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480 ''' + extra_param + '''
'''
}
}
......
examples/c/stream_demo.c
浏览文件 @ 086d6f39
......@@ -25,7 +25,7 @@ int32_t init_env() {
return -1;
}
TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
if (taos_errno(pRes) != 0) {
printf("error in create db, reason:%s\n", taos_errstr(pRes));
return -1;
......@@ -68,6 +68,14 @@ int32_t init_env() {
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create table if not exists tu3 using st1 tags(3)");
if (taos_errno(pRes) != 0) {
printf("failed to create child table tu3, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
return 0;
}
......@@ -90,10 +98,9 @@ int32_t create_stream() {
/*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/
/*const char* sql = "select sum(k) from tu1 interval(10m)";*/
/*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/
pRes = taos_query(
pConn,
"create stream stream1 trigger window_close watermark 10s into outstb as select _wstartts, sum(k) from st1 "
"interval(10s) ");
pRes = taos_query(pConn,
"create stream stream1 trigger at_once into outstb as select _wstartts, sum(k) from st1 "
"partition by tbname interval(10s) ");
if (taos_errno(pRes) != 0) {
printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes));
return -1;
......
include/common/tcommon.h
浏览文件 @ 086d6f39
......@@ -50,8 +50,10 @@ typedef enum EStreamType {
} EStreamType;
typedef struct {
SArray* pGroupList;
SArray* pTableList;
SHashObj* map; // speedup acquire the tableQueryInfo by table uid
bool needSortTableByGroupId;
void* pTagCond;
void* pTagIndexCond;
uint64_t suid;
......
include/libs/executor/executor.h
浏览文件 @ 086d6f39
......@@ -36,7 +36,7 @@ typedef struct SReadHandle {
void* vnode;
void* mnd;
SMsgCb* pMsgCb;
int8_t initTsdbReader;
// int8_t initTsdbReader;
} SReadHandle;
enum {
......
include/libs/nodes/plannodes.h
浏览文件 @ 086d6f39
......@@ -458,6 +458,7 @@ typedef struct SSubplan {
int32_t msgType; // message type for subplan, used to denote the send message type to vnode.
int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner.
char dbFName[TSDB_DB_FNAME_LEN];
char user[TSDB_USER_LEN];
SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node
SQueryNodeStat execNodeStat; // only for scan subplan
SNodeList* pChildren; // the datasource subplan,from which to fetch the result
......
include/libs/planner/planner.h
浏览文件 @ 086d6f39
......@@ -24,18 +24,19 @@ extern "C" {
#include "taos.h"
typedef struct SPlanContext {
uint64_t queryId;
int32_t acctId;
SEpSet mgmtEpSet;
SNode* pAstRoot;
bool topicQuery;
bool streamQuery;
bool rSmaQuery;
bool showRewrite;
int8_t triggerType;
int64_t watermark;
char* pMsg;
int32_t msgLen;
uint64_t queryId;
int32_t acctId;
SEpSet mgmtEpSet;
SNode* pAstRoot;
bool topicQuery;
bool streamQuery;
bool rSmaQuery;
bool showRewrite;
int8_t triggerType;
int64_t watermark;
char* pMsg;
int32_t msgLen;
const char* pUser;
} SPlanContext;
// Create the physical plan for the query, according to the AST.
......
include/libs/sync/sync.h
浏览文件 @ 086d6f39
......@@ -200,6 +200,7 @@ const char* syncGetMyRoleStr(int64_t rid);
SyncTerm syncGetMyTerm(int64_t rid);
SyncGroupId syncGetVgId(int64_t rid);
void syncGetEpSet(int64_t rid, SEpSet* pEpSet);
void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet);
int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak);
bool syncEnvIsStart();
const char* syncStr(ESyncState state);
......
source/client/src/clientImpl.c
浏览文件 @ 086d6f39
......@@ -59,7 +59,7 @@ static STscObj* taosConnectImpl(const char* user, const char* auth, const char*
SAppInstInfo* pAppInfo, int connType);
STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
uint16_t port, int connType) {
uint16_t port, int connType) {
if (taos_init() != TSDB_CODE_SUCCESS) {
return NULL;
}
......@@ -327,8 +327,8 @@ bool qnodeRequired(SRequestObj* pRequest) {
}
SAppInstInfo* pInfo = pRequest->pTscObj->pAppInfo;
bool required = false;
bool required = false;
taosThreadMutexLock(&pInfo->qnodeMutex);
required = (NULL == pInfo->pQnodeList);
taosThreadMutexUnlock(&pInfo->qnodeMutex);
......@@ -376,7 +376,8 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
.pAstRoot = pQuery->pRoot,
.showRewrite = pQuery->showRewrite,
.pMsg = pRequest->msgBuf,
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE};
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
.pUser = pRequest->pTscObj->user};
return qCreateQueryPlan(&cxt, pPlan, pNodeList);
}
......@@ -433,11 +434,11 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr
}
for (int32_t j = 0; j < vgNum; ++j) {
SVgroupInfo* pInfo = taosArrayGet(pVg, j);
SVgroupInfo* pInfo = taosArrayGet(pVg, j);
SQueryNodeLoad load = {0};
load.addr.nodeId = pInfo->vgId;
load.addr.epSet = pInfo->epSet;
taosArrayPush(nodeList, &load);
}
}
......@@ -495,17 +496,16 @@ _return:
return TSDB_CODE_SUCCESS;
}
int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SMetaData *pResultMeta) {
int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SMetaData* pResultMeta) {
SArray* pDbVgList = NULL;
SArray* pQnodeList = NULL;
int32_t code = 0;
switch (tsQueryPolicy) {
case QUERY_POLICY_VNODE: {
if (pResultMeta) {
pDbVgList = taosArrayInit(4, POINTER_BYTES);
int32_t dbNum = taosArrayGetSize(pResultMeta->pDbVgroup);
for (int32_t i = 0; i < dbNum; ++i) {
SMetaRes* pRes = taosArrayGet(pResultMeta->pDbVgroup, i);
......@@ -514,9 +514,9 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray
}
taosArrayPush(pDbVgList, &pRes->pRes);
}
}
}
code = buildVnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pDbVgList);
break;
}
......@@ -537,7 +537,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray
}
taosThreadMutexUnlock(&pInst->qnodeMutex);
}
code = buildQnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pQnodeList);
break;
}
......@@ -548,7 +548,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray
taosArrayDestroy(pDbVgList);
taosArrayDestroy(pQnodeList);
return code;
}
......@@ -556,43 +556,43 @@ int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray*
SArray* pDbVgList = NULL;
SArray* pQnodeList = NULL;
int32_t code = 0;
switch (tsQueryPolicy) {
case QUERY_POLICY_VNODE: {
int32_t dbNum = taosArrayGetSize(pRequest->dbList);
if (dbNum > 0) {
SCatalog* pCtg = NULL;
SCatalog* pCtg = NULL;
SAppInstInfo* pInst = pRequest->pTscObj->pAppInfo;
code = catalogGetHandle(pInst->clusterId, &pCtg);
if (code != TSDB_CODE_SUCCESS) {
goto _return;
}
pDbVgList = taosArrayInit(dbNum, POINTER_BYTES);
pDbVgList = taosArrayInit(dbNum, POINTER_BYTES);
SArray* pVgList = NULL;
for (int32_t i = 0; i < dbNum; ++i) {
char* dbFName = taosArrayGet(pRequest->dbList, i);
char* dbFName = taosArrayGet(pRequest->dbList, i);
SRequestConnInfo conn = {.pTrans = pInst->pTransporter,
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pInst->mgmtEp)};
.mgmtEps = getEpSet_s(&pInst->mgmtEp)};
code = catalogGetDBVgInfo(pCtg, &conn, dbFName, &pVgList);
if (code) {
goto _return;
}
taosArrayPush(pDbVgList, &pVgList);
}
}
}
code = buildVnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pDbVgList);
break;
}
case QUERY_POLICY_HYBRID:
case QUERY_POLICY_QNODE: {
getQnodeList(pRequest, &pQnodeList);
code = buildQnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pQnodeList);
break;
}
......@@ -605,11 +605,10 @@ _return:
taosArrayDestroy(pDbVgList);
taosArrayDestroy(pQnodeList);
return code;
}
int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) {
tsem_init(&schdRspSem, 0, 0);
......@@ -833,8 +832,8 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
}
}
tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64,
pRequest->self, code, tstrerror(code), pRequest->requestId);
tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code,
tstrerror(code), pRequest->requestId);
STscObj* pTscObj = pRequest->pTscObj;
if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) {
......@@ -880,7 +879,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
SArray* pNodeList = NULL;
buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList);
code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList);
taosArrayDestroy(pNodeList);
}
......@@ -935,7 +934,7 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool val
return launchQueryImpl(pRequest, pQuery, false, NULL);
}
void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultMeta) {
void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta) {
int32_t code = 0;
switch (pQuery->execMode) {
......@@ -956,7 +955,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultM
.pAstRoot = pQuery->pRoot,
.showRewrite = pQuery->showRewrite,
.pMsg = pRequest->msgBuf,
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE};
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
.pUser = pRequest->pTscObj->user};
SAppInstInfo* pAppInfo = getAppInfo(pRequest);
code = qCreateQueryPlan(&cxt, &pRequest->body.pDag, pMnodeList);
......@@ -968,7 +968,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultM
if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
SArray* pNodeList = NULL;
buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta);
SRequestConnInfo conn = {
.pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};
SSchedulerReq req = {.pConn = &conn,
......@@ -1328,7 +1328,7 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons
if (pObj) {
return pObj->id;
}
return NULL;
}
......@@ -1500,10 +1500,10 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
return TSDB_CODE_SUCCESS;
}
static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows){
static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) {
char* p = (char*)pResultInfo->pData;
int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t));
int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t));
int32_t* colLength = (int32_t*)(p + len);
len += sizeof(int32_t) * numOfCols;
......@@ -1513,7 +1513,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i
if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) {
int32_t* offset = (int32_t*)pStart;
int32_t lenTmp = numOfRows * sizeof(int32_t);
int32_t lenTmp = numOfRows * sizeof(int32_t);
len += lenTmp;
pStart += lenTmp;
......@@ -1538,7 +1538,6 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i
} else {
ASSERT(0);
}
}
} else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
int32_t lenTmp = numOfRows * sizeof(int32_t);
......@@ -1562,13 +1561,13 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
break;
}
}
if(!needConvert) return TSDB_CODE_SUCCESS;
if (!needConvert) return TSDB_CODE_SUCCESS;
char* p = (char*)pResultInfo->pData;
char* p = (char*)pResultInfo->pData;
int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows);
pResultInfo->convertJson = taosMemoryCalloc(1, dataLen);
if(pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY;
if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY;
char* p1 = pResultInfo->convertJson;
int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t));
......@@ -1637,7 +1636,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
ASSERT(0);
}
offset1[j]= len;
offset1[j] = len;
memcpy(pStart1 + len, dst, varDataTLen(dst));
len += varDataTLen(dst);
}
......@@ -1655,7 +1654,6 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
pStart += len;
pStart1 += len;
memcpy(pStart1, pStart, colLen);
}
pStart += colLen;
pStart1 += colLen1;
......@@ -1723,7 +1721,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
pStart += colLength[i];
}
if(convertUcs4){
if (convertUcs4) {
code = doConvertUCS4(pResultInfo, numOfRows, numOfCols, colLength);
}
......@@ -1840,17 +1838,18 @@ _OVER:
return code;
}
int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, int32_t len2, const char* str, int32_t acctId, char* db) {
int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, int32_t len2, const char* str,
int32_t acctId, char* db) {
SName name;
if (len1 <= 0) {
return -1;
}
const char *dbName = db;
const char *tbName = NULL;
int32_t dbLen = 0;
int32_t tbLen = 0;
const char* dbName = db;
const char* tbName = NULL;
int32_t dbLen = 0;
int32_t tbLen = 0;
if (len2 > 0) {
dbName = str + pos1;
dbLen = len1;
......@@ -1861,7 +1860,7 @@ int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, i
tbName = str + pos1;
tbLen = len1;
}
if (tNameSetDbName(&name, acctId, dbName, dbLen)) {
return -1;
}
......@@ -1881,18 +1880,18 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName,
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
bool inEscape = false;
bool inEscape = false;
int32_t code = 0;
int32_t vIdx = 0;
int32_t vPos[2];
int32_t vLen[2];
memset(vPos, -1, sizeof(vPos));
memset(vLen, 0, sizeof(vLen));
for (int32_t i = 0; ; ++i) {
for (int32_t i = 0;; ++i) {
if (0 == *(tbList + i)) {
if (vPos[vIdx] >= 0 && vLen[vIdx] <= 0) {
vLen[vIdx] = i - vPos[vIdx];
......@@ -1905,7 +1904,7 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName,
break;
}
if ('`' == *(tbList + i)) {
inEscape = !inEscape;
if (!inEscape) {
......@@ -1952,7 +1951,7 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName,
if (code) {
goto _return;
}
memset(vPos, -1, sizeof(vPos));
memset(vLen, 0, sizeof(vLen));
vIdx = 0;
......@@ -1966,8 +1965,7 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName,
continue;
}
if (('a' <= *(tbList + i) && 'z' >= *(tbList + i)) ||
('A' <= *(tbList + i) && 'Z' >= *(tbList + i)) ||
if (('a' <= *(tbList + i) && 'z' >= *(tbList + i)) || ('A' <= *(tbList + i) && 'Z' >= *(tbList + i)) ||
('0' <= *(tbList + i) && '9' >= *(tbList + i))) {
if (vLen[vIdx] > 0) {
goto _return;
......@@ -1989,32 +1987,31 @@ _return:
taosArrayDestroy(*pReq);
*pReq = NULL;
return terrno;
}
void syncCatalogFn(SMetaData* pResult, void* param, int32_t code) {
SSyncQueryParam *pParam = param;
SSyncQueryParam* pParam = param;
pParam->pRequest->code = code;
tsem_post(&pParam->sem);
}
void syncQueryFn(void *param, void *res, int32_t code) {
SSyncQueryParam *pParam = param;
void syncQueryFn(void* param, void* res, int32_t code) {
SSyncQueryParam* pParam = param;
pParam->pRequest = res;
pParam->pRequest->code = code;
tsem_post(&pParam->sem);
}
void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param, bool validateOnly) {
STscObj *pTscObj = acquireTscObj(*(int64_t *)taos);
void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) {
STscObj* pTscObj = acquireTscObj(*(int64_t*)taos);
if (pTscObj == NULL || sql == NULL || NULL == fp) {
terrno = TSDB_CODE_INVALID_PARA;
if (pTscObj) {
releaseTscObj(*(int64_t *)taos);
releaseTscObj(*(int64_t*)taos);
} else {
terrno = TSDB_CODE_TSC_DISCONNECTED;
}
......@@ -2031,7 +2028,7 @@ void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void
return;
}
SRequestObj *pRequest = NULL;
SRequestObj* pRequest = NULL;
int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
......@@ -2045,45 +2042,41 @@ void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void
doAsyncQuery(pRequest, false);
}
TAOS_RES *taosQueryImpl(TAOS *taos, const char *sql, bool validateOnly) {
TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
if (NULL == taos) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL;
}
STscObj *pTscObj = acquireTscObj(*(int64_t *)taos);
STscObj* pTscObj = acquireTscObj(*(int64_t*)taos);
if (pTscObj == NULL || sql == NULL) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL;
}
#if SYNC_ON_TOP_OF_ASYNC
SSyncQueryParam *param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
tsem_init(&param->sem, 0, 0);
taosAsyncQueryImpl(taos, sql, syncQueryFn, param, validateOnly);
tsem_wait(&param->sem);
releaseTscObj(*(int64_t *)taos);
releaseTscObj(*(int64_t*)taos);
return param->pRequest;
#else
size_t sqlLen = strlen(sql);
if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) {
releaseTscObj(*(int64_t *)taos);
releaseTscObj(*(int64_t*)taos);
tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
return NULL;
}
TAOS_RES *pRes = execQuery(pTscObj, sql, sqlLen, validateOnly);
TAOS_RES* pRes = execQuery(pTscObj, sql, sqlLen, validateOnly);
releaseTscObj(*(int64_t *)taos);
releaseTscObj(*(int64_t*)taos);
return pRes;
#endif
}
source/common/src/tglobal.c
浏览文件 @ 086d6f39
......@@ -963,6 +963,10 @@ int32_t taosSetCfg(SConfig *pCfg, char* name) {
tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32;
} else if (strcasecmp("transPullupInterval", name) == 0) {
tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
} else if (strcasecmp("ttlUnit", name) == 0) {
tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32;
} else if (strcasecmp("ttlPushInterval", name) == 0) {
tsTtlPushInterval = cfgGetItem(pCfg, "ttlPushInterval")->i32;
} else if (strcasecmp("tmrDebugFlag", name) == 0) {
tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32;
} else if (strcasecmp("tsdbDebugFlag", name) == 0) {
......
source/dnode/mgmt/node_util/src/dmFile.c
浏览文件 @ 086d6f39
......@@ -133,10 +133,10 @@ TdFilePtr dmCheckRunning(const char *dataDir) {
ret = taosLockFile(pFile);
if (ret == 0) break;
terrno = TAOS_SYSTEM_ERROR(errno);
taosMsleep(100);
taosMsleep(1000);
retryTimes++;
dError("failed to lock file:%s since %s, retryTimes:%d", filepath, terrstr(), retryTimes);
} while (retryTimes < 120);
} while (retryTimes < 12);
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
......
source/dnode/mnode/impl/inc/mndAuth.h
浏览文件 @ 086d6f39
......@@ -23,13 +23,15 @@ extern "C" {
#endif
typedef enum {
MND_OPER_CREATE_USER = 1,
MND_OPER_CONNECT = 1,
MND_OPER_CREATE_USER,
MND_OPER_DROP_USER,
MND_OPER_ALTER_USER,
MND_OPER_CREATE_BNODE,
MND_OPER_DROP_BNODE,
MND_OPER_CREATE_DNODE,
MND_OPER_DROP_DNODE,
MND_OPER_CONFIG_DNODE,
MND_OPER_CREATE_MNODE,
MND_OPER_DROP_MNODE,
MND_OPER_CREATE_QNODE,
......@@ -37,6 +39,7 @@ typedef enum {
MND_OPER_CREATE_SNODE,
MND_OPER_DROP_SNODE,
MND_OPER_REDISTRIBUTE_VGROUP,
MND_OPER_MERGE_VGROUP,
MND_OPER_SPLIT_VGROUP,
MND_OPER_BALANCE_VGROUP,
MND_OPER_CREATE_FUNC,
......
source/dnode/mnode/impl/src/mndAuth.c
浏览文件 @ 086d6f39
......@@ -93,8 +93,13 @@ int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) {
goto _OVER;
}
terrno = TSDB_CODE_MND_NO_RIGHTS;
code = -1;
switch (operType) {
case MND_OPER_CONNECT:
break;
default:
terrno = TSDB_CODE_MND_NO_RIGHTS;
code = -1;
}
_OVER:
mndReleaseUser(pMnode, pUser);
......@@ -102,7 +107,13 @@ _OVER:
}
int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) {
if (pUser->superUser && pAlter->alterType != TSDB_ALTER_USER_PASSWD) {
terrno = TSDB_CODE_MND_NO_RIGHTS;
return -1;
}
if (pOperUser->superUser) return 0;
if (!pOperUser->enable) {
terrno = TSDB_CODE_MND_USER_DISABLED;
return -1;
......
source/dnode/mnode/impl/src/mndBnode.c
浏览文件 @ 086d6f39
......@@ -277,6 +277,9 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) {
}
mDebug("bnode:%d, start to create", createReq.dnodeId);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) {
goto _OVER;
}
pObj = mndAcquireBnode(pMnode, createReq.dnodeId);
if (pObj != NULL) {
......@@ -292,10 +295,6 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) {
goto _OVER;
}
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) {
goto _OVER;
}
code = mndCreateBnode(pMnode, pReq, pDnode, &createReq);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
......@@ -383,6 +382,9 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) {
}
mDebug("bnode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) {
goto _OVER;
}
if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG;
......@@ -394,10 +396,6 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) {
goto _OVER;
}
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) {
goto _OVER;
}
code = mndDropBnode(pMnode, pReq, pObj);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
......
source/dnode/mnode/impl/src/mndDnode.c
浏览文件 @ 086d6f39
......@@ -609,7 +609,6 @@ _OVER:
return code;
}
static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
......@@ -622,6 +621,9 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
}
mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) {
goto _OVER;
}
if (createReq.fqdn[0] == 0 || createReq.port <= 0 || createReq.port > UINT16_MAX) {
terrno = TSDB_CODE_MND_INVALID_DNODE_EP;
......@@ -635,10 +637,6 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
goto _OVER;
}
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) {
goto _OVER;
}
code = mndCreateDnode(pMnode, pReq, &createReq);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
......@@ -717,6 +715,9 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
}
mInfo("dnode:%d, start to drop, ep:%s:%d", dropReq.dnodeId, dropReq.fqdn, dropReq.port);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) {
goto _OVER;
}
pDnode = mndAcquireDnode(pMnode, dropReq.dnodeId);
if (pDnode == NULL) {
......@@ -753,10 +754,6 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
}
}
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) {
goto _OVER;
}
code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
......@@ -782,6 +779,10 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
}
mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) {
return -1;
}
SDnodeObj *pDnode = mndAcquireDnode(pMnode, cfgReq.dnodeId);
if (pDnode == NULL) {
mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr());
......
source/dnode/mnode/impl/src/mndFunc.c
浏览文件 @ 086d6f39
......@@ -283,6 +283,9 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
}
mDebug("func:%s, start to create", createReq.name);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) {
goto _OVER;
}
pFunc = mndAcquireFunc(pMnode, createReq.name);
if (pFunc != NULL) {
......@@ -318,10 +321,6 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
goto _OVER;
}
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) {
goto _OVER;
}
code = mndCreateFunc(pMnode, pReq, &createReq);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
......@@ -347,6 +346,9 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
}
mDebug("func:%s, start to drop", dropReq.name);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) {
goto _OVER;
}
if (dropReq.name[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_FUNC_NAME;
......@@ -365,10 +367,6 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
}
}
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) {
goto _OVER;
}
code = mndDropFunc(pMnode, pReq, pFunc);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
......
source/dnode/mnode/impl/src/mndMain.c
浏览文件 @ 086d6f39
......@@ -100,7 +100,7 @@ static void *mndThreadFp(void *param) {
taosMsleep(100);
if (mndGetStop(pMnode)) break;
if (lastTime % (tsTransPullupInterval * 10) == 1) {
if (lastTime % (tsTtlPushInterval * 10) == 1) {
mndTtlTimer(pMnode);
}
......@@ -529,24 +529,33 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
if (!IsReq(pMsg)) return 0;
if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0;
if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
pMsg->msgType == TDMT_MND_TRANS_TIMER || TDMT_MND_TTL_TIMER) {
pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER) {
return -1;
}
const STraceId *trace = &pMsg->info.traceId;
mError("msg:%p, failed to check mnode state since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType));
SEpSet epSet = {0};
mndGetMnodeEpSet(pMsg->info.node, &epSet);
int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
pMsg->info.rsp = rpcMallocCont(contLen);
if (pMsg->info.rsp != NULL) {
tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet);
pMsg->info.rspLen = contLen;
terrno = TSDB_CODE_RPC_REDIRECT;
const STraceId *trace = &pMsg->info.traceId;
mError("msg:%p, failed to check mnode state since %s, type:%s, numOfMnodes:%d inUse:%d", pMsg, terrstr(),
TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse);
if (epSet.numOfEps > 0) {
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
mInfo("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
}
int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
pMsg->info.rsp = rpcMallocCont(contLen);
if (pMsg->info.rsp != NULL) {
tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet);
pMsg->info.rspLen = contLen;
terrno = TSDB_CODE_RPC_REDIRECT;
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
terrno = TSDB_CODE_APP_NOT_READY;
}
return -1;
......@@ -555,10 +564,10 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
static int32_t mndCheckMsgContent(SRpcMsg *pMsg) {
if (!IsReq(pMsg)) return 0;
if (pMsg->contLen != 0 && pMsg->pCont != NULL) return 0;
const STraceId *trace = &pMsg->info.traceId;
mGError("msg:%p, failed to check msg, cont:%p contLen:%d, app:%p type:%s", pMsg, pMsg->pCont, pMsg->contLen,
pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
terrno = TSDB_CODE_INVALID_MSG_LEN;
return -1;
}
......@@ -723,7 +732,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pStb);
if (pIter == NULL) break;
SMonStbDesc desc = {0};
SMonStbDesc desc = {0};
SName name1 = {0};
tNameFromString(&name1, pStb->db, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
......
source/dnode/mnode/impl/src/mndMnode.c
浏览文件 @ 086d6f39
......@@ -389,6 +389,9 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
}
mDebug("mnode:%d, start to create", createReq.dnodeId);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) {
goto _OVER;
}
pObj = mndAcquireMnode(pMnode, createReq.dnodeId);
if (pObj != NULL) {
......@@ -414,10 +417,6 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
goto _OVER;
}
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) {
goto _OVER;
}
code = mndCreateMnode(pMnode, pReq, pDnode, &createReq);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
......@@ -495,7 +494,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
{
int32_t contLen = tSerializeSSetStandbyReq(NULL, 0, &standbyReq) + sizeof(SMsgHead);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSSetStandbyReq((char*)pReq + sizeof(SMsgHead), contLen, &standbyReq);
tSerializeSSetStandbyReq((char *)pReq + sizeof(SMsgHead), contLen, &standbyReq);
SMsgHead *pHead = pReq;
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(MNODE_HANDLE);
......@@ -595,6 +594,9 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
}
mDebug("mnode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) {
goto _OVER;
}
if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG;
......@@ -621,10 +623,6 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
goto _OVER;
}
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) {
goto _OVER;
}
code = mndDropMnode(pMnode, pReq, pObj);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
......
source/dnode/mnode/impl/src/mndProfile.c
浏览文件 @ 086d6f39
......@@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE
#include "mndProfile.h"
#include "mndAuth.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
......@@ -217,36 +218,42 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
SConnObj *pConn = NULL;
int32_t code = -1;
SConnectReq connReq = {0};
char ip[30] = {0};
char ip[24] = {0};
const STraceId *trace = &pReq->info.traceId;
if (tDeserializeSConnectReq(pReq->pCont, pReq->contLen, &connReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto CONN_OVER;
goto _OVER;
}
taosIp2String(pReq->info.conn.clientIp, ip);
pUser = mndAcquireUser(pMnode, pReq->info.conn.user);
if (pUser == NULL) {
mGError("user:%s, failed to login while acquire user since %s", pReq->info.conn.user, terrstr());
goto CONN_OVER;
mGError("user:%s, failed to login from %s while acquire user since %s", pReq->info.conn.user, ip, terrstr());
goto _OVER;
}
if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) {
mGError("user:%s, failed to auth while acquire user, input:%s", pReq->info.conn.user, connReq.passwd);
if (strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1) != 0) {
mGError("user:%s, failed to login from %s since invalid pass, input:%s", pReq->info.conn.user, ip, connReq.passwd);
code = TSDB_CODE_RPC_AUTH_FAILURE;
goto CONN_OVER;
goto _OVER;
}
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) {
mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr());
goto _OVER;
}
if (connReq.db[0]) {
char db[TSDB_DB_FNAME_LEN];
char db[TSDB_DB_FNAME_LEN] = {0};
snprintf(db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db);
pDb = mndAcquireDb(pMnode, db);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_INVALID_DB;
mGError("user:%s, failed to login from %s while use db:%s since %s", pReq->info.conn.user, ip, connReq.db,
terrstr());
goto CONN_OVER;
goto _OVER;
}
}
......@@ -254,7 +261,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
pReq->info.conn.clientPort, connReq.pid, connReq.app, connReq.startTime);
if (pConn == NULL) {
mGError("user:%s, failed to login from %s while create connection since %s", pReq->info.conn.user, ip, terrstr());
goto CONN_OVER;
goto _OVER;
}
SConnectRsp connectRsp = {0};
......@@ -271,9 +278,9 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
mndGetMnodeEpSet(pMnode, &connectRsp.epSet);
int32_t contLen = tSerializeSConnectRsp(NULL, 0, &connectRsp);
if (contLen < 0) goto CONN_OVER;
if (contLen < 0) goto _OVER;
void *pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) goto CONN_OVER;
if (pRsp == NULL) goto _OVER;
tSerializeSConnectRsp(pRsp, contLen, &connectRsp);
pReq->info.rspLen = contLen;
......@@ -283,7 +290,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
code = 0;
CONN_OVER:
_OVER:
mndReleaseUser(pMnode, pUser);
mndReleaseDb(pMnode, pDb);
......
source/dnode/mnode/impl/src/mndQnode.c
浏览文件 @ 086d6f39
......@@ -279,6 +279,9 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
}
mDebug("qnode:%d, start to create", createReq.dnodeId);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) {
goto _OVER;
}
pObj = mndAcquireQnode(pMnode, createReq.dnodeId);
if (pObj != NULL) {
......@@ -294,10 +297,6 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
goto _OVER;
}
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) {
goto _OVER;
}
code = mndCreateQnode(pMnode, pReq, pDnode, &createReq);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
......@@ -391,6 +390,9 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
}
mDebug("qnode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) {
goto _OVER;
}
if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG;
......@@ -402,10 +404,6 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
goto _OVER;
}
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) {
goto _OVER;
}
code = mndDropQnode(pMnode, pReq, pObj);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
......@@ -418,19 +416,19 @@ _OVER:
return code;
}
int32_t mndCreateQnodeList(SMnode *pMnode, SArray** pList, int32_t limit) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SQnodeObj *pObj = NULL;
int32_t numOfRows = 0;
int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SQnodeObj *pObj = NULL;
int32_t numOfRows = 0;
SArray* qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad));
SArray *qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad));
if (NULL == qnodeList) {
mError("failed to alloc epSet while process qnode list req");
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
while (1) {
pIter = sdbFetch(pSdb, SDB_QNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
......@@ -457,7 +455,6 @@ int32_t mndCreateQnodeList(SMnode *pMnode, SArray** pList, int32_t limit)
return TSDB_CODE_SUCCESS;
}
static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) {
int32_t code = -1;
SMnode *pMnode = pReq->info.node;
......
source/dnode/mnode/impl/src/mndSnode.c
浏览文件 @ 086d6f39
......@@ -285,6 +285,9 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
}
mDebug("snode:%d, start to create", createReq.dnodeId);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) {
goto _OVER;
}
pObj = mndAcquireSnode(pMnode, createReq.dnodeId);
if (pObj != NULL) {
......@@ -300,10 +303,6 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
goto _OVER;
}
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) {
goto _OVER;
}
code = mndCreateSnode(pMnode, pReq, pDnode, &createReq);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
......@@ -398,6 +397,9 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
}
mDebug("snode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) {
goto _OVER;
}
if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG;
......@@ -409,10 +411,6 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
goto _OVER;
}
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) {
goto _OVER;
}
// check deletable
code = mndDropSnode(pMnode, pReq, pObj);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
......
source/dnode/mnode/impl/src/mndSync.c
浏览文件 @ 086d6f39
......@@ -198,6 +198,10 @@ int32_t mndInitSync(SMnode *pMnode) {
return -1;
}
// decrease election timer
setElectTimerMS(pMgmt->sync, 600);
setHeartbeatTimerMS(pMgmt->sync, 300);
mDebug("mnode-sync is opened, id:%" PRId64, pMgmt->sync);
return 0;
}
......@@ -261,7 +265,8 @@ bool mndIsMaster(SMnode *pMnode) {
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
if (!syncIsReady(pMgmt->sync)) {
terrno = TSDB_CODE_SYN_NOT_LEADER;
// get terrno from syncIsReady
// terrno = TSDB_CODE_SYN_NOT_LEADER;
return false;
}
......
source/dnode/mnode/impl/src/mndUser.c
浏览文件 @ 086d6f39
......@@ -295,7 +295,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate
tstrncpy(userObj.acct, acct, TSDB_USER_LEN);
userObj.createdTime = taosGetTimestampMs();
userObj.updateTime = userObj.createdTime;
userObj.superUser = pCreate->superUser;
userObj.superUser = 0;//pCreate->superUser;
userObj.sysInfo = pCreate->sysInfo;
userObj.enable = pCreate->enable;
......
source/dnode/mnode/impl/src/mndVgroup.c
浏览文件 @ 086d6f39
......@@ -1212,8 +1212,9 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) {
}
mInfo("vgId:%d, start to redistribute vgroup to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) goto _OVER;
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) {
goto _OVER;
}
pVgroup = mndAcquireVgroup(pMnode, req.vgId);
if (pVgroup == NULL) goto _OVER;
......@@ -1506,6 +1507,9 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) {
SDbObj *pDb = NULL;
mDebug("vgId:%d, start to split", vgId);
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) {
goto _OVER;
}
pVgroup = mndAcquireVgroup(pMnode, vgId);
if (pVgroup == NULL) goto _OVER;
......@@ -1513,8 +1517,6 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) {
pDb = mndAcquireDb(pMnode, pVgroup->dbName);
if (pDb == NULL) goto _OVER;
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) goto _OVER;
code = mndSplitVgroup(pMnode, pReq, pDb, pVgroup);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
......@@ -1655,8 +1657,9 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) {
}
mInfo("start to balance vgroup");
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) goto _OVER;
if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) {
goto _OVER;
}
while (1) {
SDnodeObj *pDnode = NULL;
......
source/dnode/mnode/impl/test/user/user.cpp
浏览文件 @ 086d6f39
......@@ -33,6 +33,8 @@ TEST_F(MndTestUser, 01_Show_User) {
TEST_F(MndTestUser, 02_Create_User) {
{
SCreateUserReq createReq = {0};
createReq.enable = 1;
createReq.sysInfo = 1;
strcpy(createReq.user, "");
strcpy(createReq.pass, "p1");
......@@ -47,6 +49,8 @@ TEST_F(MndTestUser, 02_Create_User) {
{
SCreateUserReq createReq = {0};
createReq.enable = 1;
createReq.sysInfo = 1;
strcpy(createReq.user, "u1");
strcpy(createReq.pass, "");
......@@ -61,6 +65,8 @@ TEST_F(MndTestUser, 02_Create_User) {
{
SCreateUserReq createReq = {0};
createReq.enable = 1;
createReq.sysInfo = 1;
strcpy(createReq.user, "root");
strcpy(createReq.pass, "1");
......@@ -75,6 +81,8 @@ TEST_F(MndTestUser, 02_Create_User) {
{
SCreateUserReq createReq = {0};
createReq.enable = 1;
createReq.sysInfo = 1;
strcpy(createReq.user, "u1");
strcpy(createReq.pass, "p1");
......@@ -108,9 +116,11 @@ TEST_F(MndTestUser, 02_Create_User) {
{
SCreateUserReq createReq = {0};
createReq.enable = 1;
createReq.sysInfo = 1;
strcpy(createReq.user, "u2");
strcpy(createReq.pass, "p1");
createReq.superUser = 1;
createReq.superUser = 0;
int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq);
void* pReq = rpcMallocCont(contLen);
......@@ -144,9 +154,11 @@ TEST_F(MndTestUser, 02_Create_User) {
TEST_F(MndTestUser, 03_Alter_User) {
{
SCreateUserReq createReq = {0};
createReq.enable = 1;
createReq.sysInfo = 1;
strcpy(createReq.user, "u3");
strcpy(createReq.pass, "p1");
createReq.superUser = 1;
createReq.superUser = 0;
int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq);
void* pReq = rpcMallocCont(contLen);
......@@ -225,7 +237,7 @@ TEST_F(MndTestUser, 03_Alter_User) {
alterReq.alterType = TSDB_ALTER_USER_SUPERUSER;
strcpy(alterReq.user, "u3");
strcpy(alterReq.pass, "1");
alterReq.superUser = 1;
alterReq.superUser = 0;
int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq);
void* pReq = rpcMallocCont(contLen);
......@@ -361,7 +373,7 @@ TEST_F(MndTestUser, 03_Alter_User) {
SGetUserAuthRsp authRsp = {0};
tDeserializeSGetUserAuthRsp(pRsp->pCont, pRsp->contLen, &authRsp);
EXPECT_STREQ(authRsp.user, "u3");
EXPECT_EQ(authRsp.superAuth, 1);
EXPECT_EQ(authRsp.superAuth, 0);
int32_t numOfReadDbs = taosHashGetSize(authRsp.readDbs);
int32_t numOfWriteDbs = taosHashGetSize(authRsp.writeDbs);
EXPECT_EQ(numOfReadDbs, 1);
......@@ -436,6 +448,8 @@ TEST_F(MndTestUser, 05_Drop_User) {
{
SCreateUserReq createReq = {0};
createReq.enable = 1;
createReq.sysInfo = 1;
strcpy(createReq.user, "u1");
strcpy(createReq.pass, "p1");
......@@ -468,6 +482,8 @@ TEST_F(MndTestUser, 05_Drop_User) {
TEST_F(MndTestUser, 06_Create_Drop_Alter_User) {
{
SCreateUserReq createReq = {0};
createReq.enable = 1;
createReq.sysInfo = 1;
strcpy(createReq.user, "u1");
strcpy(createReq.pass, "p1");
......@@ -482,6 +498,8 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) {
{
SCreateUserReq createReq = {0};
createReq.enable = 1;
createReq.sysInfo = 1;
strcpy(createReq.user, "u2");
strcpy(createReq.pass, "p2");
......
source/dnode/vnode/inc/vnode.h
浏览文件 @ 086d6f39
......@@ -116,7 +116,8 @@ typedef void *tsdbReaderT;
#define BLOCK_LOAD_TABLE_SEQ_ORDER 2
#define BLOCK_LOAD_TABLE_RR_ORDER 3
tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *tableInfoGroup, uint64_t qId,
int32_t tsdbSetTableList(tsdbReaderT reader, SArray* tableList);
tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *tableList, uint64_t qId,
uint64_t taskId);
tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *groupList, uint64_t qId,
void *pMemRef);
......@@ -195,7 +196,6 @@ struct SVnodeCfg {
typedef struct {
TSKEY lastKey;
uint64_t uid;
uint64_t groupId;
} STableKeyInfo;
struct SMetaEntry {
......
source/dnode/vnode/src/inc/vnodeInt.h
浏览文件 @ 086d6f39
......@@ -121,7 +121,7 @@ int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSub
int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock,
SSubmitBlkRsp* pRsp);
int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* tableList, uint64_t qId,
uint64_t taskId);
tsdbReaderT tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
void* pMemRef);
......
source/dnode/vnode/src/meta/metaTable.c
浏览文件 @ 086d6f39
......@@ -381,6 +381,7 @@ int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) {
for (int i = 0; i < taosArrayGetSize(tbUids); ++i) {
tb_uid_t *uid = (tb_uid_t *)taosArrayGet(tbUids, i);
metaDropTableByUid(pMeta, *uid, NULL);
metaDebug("ttl drop table:%"PRId64, *uid);
}
metaULock(pMeta);
return 0;
......@@ -443,7 +444,6 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
// drop schema.db (todo)
}
metaError("ttl drop table:%s", e.name);
tDecoderClear(&dc);
tdbFree(pData);
......@@ -976,7 +976,9 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
SDecoder dc = {0};
// get super table
tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData);
if(tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0){
return -1;
}
tbDbKey.uid = pCtbEntry->ctbEntry.suid;
tbDbKey.version = *(int64_t *)pData;
tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData);
......
source/dnode/vnode/src/tq/tq.c
浏览文件 @ 086d6f39
......@@ -403,7 +403,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
.reader = pHandle->execHandle.pExecReader[i],
.meta = pTq->pVnode->pMeta,
.vnode = pTq->pVnode,
.initTsdbReader = 1,
// .initTsdbReader = 1,
};
pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle);
ASSERT(pHandle->execHandle.execCol.task[i]);
......@@ -479,7 +479,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
.reader = pStreamReader,
.meta = pTq->pVnode->pMeta,
.vnode = pTq->pVnode,
.initTsdbReader = 1,
// .initTsdbReader = 1,
};
/*pTask->exec.inputHandle = pStreamReader;*/
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
......
source/dnode/vnode/src/tq/tqRead.c
浏览文件 @ 086d6f39
......@@ -243,7 +243,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_
if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
break;
}
if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) {
if (colDataAppend(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) {
goto FAIL;
}
}
......
source/dnode/vnode/src/tsdb/tsdbRead.c
浏览文件 @ 086d6f39
......@@ -223,9 +223,8 @@ int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) {
return rows;
}
static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableListInfo* pTableList) {
size_t tableSize = taosArrayGetSize(pTableList->pTableList);
assert(tableSize >= 1);
static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, SArray* pTableList) {
size_t tableSize = taosArrayGetSize(pTableList);
// allocate buffer in order to load data blocks from file
SArray* pTableCheckInfo = taosArrayInit(tableSize, sizeof(STableCheckInfo));
......@@ -235,7 +234,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S
// todo apply the lastkey of table check to avoid to load header file
for (int32_t j = 0; j < tableSize; ++j) {
STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pTableList->pTableList, j);
STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pTableList, j);
STableCheckInfo info = {.lastKey = pKeyInfo->lastKey, .tableId = pKeyInfo->uid};
info.suid = pTsdbReadHandle->suid;
......@@ -254,8 +253,6 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S
pTsdbReadHandle->idStr);
}
// TODO group table according to the tag value.
taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar);
return pTableCheckInfo;
}
......@@ -497,8 +494,21 @@ static int32_t setCurrentSchema(SVnode* pVnode, STsdbReadHandle* pTsdbReadHandle
return TSDB_CODE_SUCCESS;
}
tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
int32_t tsdbSetTableList(tsdbReaderT reader, SArray* tableList){
STsdbReadHandle* pTsdbReadHandle = reader;
if(pTsdbReadHandle->pTableCheckInfo) taosArrayDestroy(pTsdbReadHandle->pTableCheckInfo);
pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, tableList);
if (pTsdbReadHandle->pTableCheckInfo == NULL) {
return TSDB_CODE_TDB_OUT_OF_MEMORY;
}
return TDB_CODE_SUCCESS;
}
tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* tableList, uint64_t qId,
uint64_t taskId) {
if(taosArrayGetSize(tableList) == 0){
return NULL;
}
STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(pVnode, pCond, qId, taskId);
if (pTsdbReadHandle == NULL) {
return NULL;
......@@ -543,7 +553,7 @@ tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableLis
}
tsdbDebug("%p total numOfTable:%" PRIzu " in this query, table %" PRIzu " %s", pTsdbReadHandle,
taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), taosArrayGetSize(tableList->pTableList),
taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), taosArrayGetSize(tableList),
pTsdbReadHandle->idStr);
return (tsdbReaderT)pTsdbReadHandle;
......@@ -639,7 +649,7 @@ tsdbReaderT tsdbQueryLastRow(SVnode* pVnode, SQueryTableDataCond* pCond, STableL
return NULL;
}
STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)tsdbReaderOpen(pVnode, pCond, pList, qId, taskId);
STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)tsdbReaderOpen(pVnode, pCond, pList->pTableList, qId, taskId);
if (pTsdbReadHandle == NULL) {
return NULL;
}
......@@ -2842,7 +2852,7 @@ int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) {
break;
}
STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id, .groupId = 0};
STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id};
taosArrayPush(list, &info);
}
......@@ -3644,17 +3654,6 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) {
}
}
static int tsdbCheckInfoCompar(const void* key1, const void* key2) {
if (((STableCheckInfo*)key1)->tableId < ((STableCheckInfo*)key2)->tableId) {
return -1;
} else if (((STableCheckInfo*)key1)->tableId > ((STableCheckInfo*)key2)->tableId) {
return 1;
} else {
ASSERT(false);
return 0;
}
}
static void* doFreeColumnInfoData(SArray* pColumnInfoData) {
if (pColumnInfoData == NULL) {
return NULL;
......
source/dnode/vnode/src/vnd/vnodeSync.c
浏览文件 @ 086d6f39
......@@ -144,11 +144,15 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
vnodeAccumBlockMsg(pVnode, pMsg->msgType);
} else if (code == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) {
SEpSet newEpSet = {0};
syncGetRetryEpSet(pVnode->sync, &newEpSet);
/*
syncGetEpSet(pVnode->sync, &newEpSet);
SEp *pEp = &newEpSet.eps[newEpSet.inUse];
if (pEp->port == tsServerPort && strcmp(pEp->fqdn, tsLocalFqdn) == 0) {
newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps;
}
*/
vGTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", vgId, pMsg, newEpSet.numOfEps,
newEpSet.inUse);
......
source/libs/executor/inc/executorimpl.h
浏览文件 @ 086d6f39
......@@ -273,6 +273,10 @@ typedef struct STableScanInfo {
SSampleExecInfo sample; // sample execution info
int32_t curTWinIdx;
int32_t currentGroupId;
uint64_t queryId;
uint64_t taskId;
} STableScanInfo;
typedef struct STagScanInfo {
......@@ -706,7 +710,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId);
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo);
......@@ -749,8 +753,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, SBlockDistScanPhysiNode* pBlockScanNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle,
STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup);
SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle,
STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId);
SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, bool multigroupResult,
SExecTaskInfo* pTaskInfo);
......@@ -845,7 +849,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey);
int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey);
#ifdef __cplusplus
}
......
source/libs/executor/src/executil.c
浏览文件 @ 086d6f39
......@@ -287,6 +287,7 @@ static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){
int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) {
int32_t code = TSDB_CODE_SUCCESS;
pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo));
if(pListInfo->pTableList == NULL) return TSDB_CODE_OUT_OF_MEMORY;
uint64_t tableUid = pScanNode->uid;
......@@ -314,7 +315,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
}
for (int i = 0; i < taosArrayGetSize(res); i++) {
STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0};
STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i)};
taosArrayPush(pListInfo->pTableList, &info);
}
taosArrayDestroy(res);
......@@ -335,9 +336,14 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
}
}
}else { // Create one table group.
STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0};
STableKeyInfo info = {.lastKey = 0, .uid = tableUid};
taosArrayPush(pListInfo->pTableList, &info);
}
pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES);
if(pListInfo->pGroupList == NULL) return TSDB_CODE_OUT_OF_MEMORY;
//put into list as default group, remove it if grouping sorting is required later
taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList);
return code;
}
......
source/libs/executor/src/executorimpl.c
浏览文件 @ 086d6f39
......@@ -3862,9 +3862,6 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT
return pTaskInfo;
}
static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId);
static SArray* extractColumnInfo(SNodeList* pNodeList);
int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) {
......@@ -3895,8 +3892,67 @@ int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskI
return TSDB_CODE_SUCCESS;
}
int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey) {
if (groupKey == NULL) {
static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum){
taosArrayClear(pTableListInfo->pGroupList);
SArray *sortSupport = taosArrayInit(groupNum, sizeof(uint64_t));
if(sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY;
for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) {
STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
uint64_t* groupId = taosHashGet(pTableListInfo->map, &info->uid, sizeof(uint64_t));
int32_t index = taosArraySearchIdx(sortSupport, groupId, compareUint64Val, TD_EQ);
if (index == -1){
void *p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT);
SArray *tGroup = taosArrayInit(8, sizeof(STableKeyInfo));
if(tGroup == NULL) {
taosArrayDestroy(sortSupport);
return TSDB_CODE_OUT_OF_MEMORY;
}
if(taosArrayPush(tGroup, info) == NULL){
qError("taos push info array error");
taosArrayDestroy(sortSupport);
return TSDB_CODE_QRY_APP_ERROR;
}
if(p == NULL){
if(taosArrayPush(sortSupport, groupId) != NULL){
qError("taos push support array error");
taosArrayDestroy(sortSupport);
return TSDB_CODE_QRY_APP_ERROR;
}
if(taosArrayPush(pTableListInfo->pGroupList, &tGroup) != NULL){
qError("taos push group array error");
taosArrayDestroy(sortSupport);
return TSDB_CODE_QRY_APP_ERROR;
}
}else{
int32_t pos = TARRAY_ELEM_IDX(sortSupport, p);
if(taosArrayInsert(sortSupport, pos, groupId) == NULL){
qError("taos insert support array error");
taosArrayDestroy(sortSupport);
return TSDB_CODE_QRY_APP_ERROR;
}
if(taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL){
qError("taos insert group array error");
taosArrayDestroy(sortSupport);
return TSDB_CODE_QRY_APP_ERROR;
}
}
}else{
SArray* tGroup = (SArray*)taosArrayGetP(pTableListInfo->pGroupList, index);
if(taosArrayPush(tGroup, info) == NULL){
qError("taos push uid array error");
taosArrayDestroy(sortSupport);
return TSDB_CODE_QRY_APP_ERROR;
}
}
}
taosArrayDestroy(sortSupport);
return TDB_CODE_SUCCESS;
}
int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group) {
if (group == NULL) {
return TDB_CODE_SUCCESS;
}
......@@ -3906,13 +3962,14 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
}
int32_t keyLen = 0;
void* keyBuf = NULL;
int32_t numOfGroupCols = taosArrayGetSize(groupKey);
for (int32_t j = 0; j < numOfGroupCols; ++j) {
SColumn* pCol = taosArrayGet(groupKey, j);
keyLen += pCol->bytes; // actual data + null_flag
SNode* node;
FOREACH(node, group) {
SExprNode *pExpr = (SExprNode *)node;
keyLen += pExpr->resType.bytes;
}
int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols;
int32_t nullFlagSize = sizeof(int8_t) * LIST_LENGTH(group);
keyLen += nullFlagSize;
keyBuf = taosMemoryCalloc(1, keyLen);
......@@ -3920,59 +3977,68 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t groupNum = 0;
for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) {
STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
SMetaReader mr = {0};
metaReaderInit(&mr, pHandle->meta, 0);
metaGetTableEntryByUid(&mr, info->uid);
SNodeList *groupNew = nodesCloneList(group);
nodesRewriteExprsPostOrder(groupNew, doTranslateTagExpr, &mr);
char* isNull = (char*)keyBuf;
char* pStart = (char*)keyBuf + sizeof(int8_t) * numOfGroupCols;
for (int32_t j = 0; j < numOfGroupCols; ++j) {
SColumn* pCol = taosArrayGet(groupKey, j);
if (strcmp(pCol->name, "tbname") == 0) {
isNull[i] = 0;
memcpy(pStart, mr.me.name, strlen(mr.me.name));
pStart += strlen(mr.me.name);
char* pStart = (char*)keyBuf + nullFlagSize;
SNode* pNode;
int32_t index = 0;
FOREACH(pNode, groupNew){
SNode* pNew = NULL;
int32_t code = scalarCalculateConstants(pNode, &pNew);
if (TSDB_CODE_SUCCESS == code) {
REPLACE_NODE(pNew);
} else {
STagVal tagVal = {0};
tagVal.cid = pCol->colId;
const char* p = metaGetTableTagVal(&mr.me, pCol->type, &tagVal);
if (p == NULL) {
isNull[j] = 1;
continue;
}
isNull[i] = 0;
if (pCol->type == TSDB_DATA_TYPE_JSON) {
// int32_t dataLen = getJsonValueLen(pkey->pData);
// memcpy(pStart, (pkey->pData), dataLen);
// pStart += dataLen;
} else if (IS_VAR_DATA_TYPE(pCol->type)) {
memcpy(pStart, tagVal.pData, tagVal.nData);
pStart += tagVal.nData;
ASSERT(tagVal.nData <= pCol->bytes);
} else {
memcpy(pStart, &(tagVal.i64), pCol->bytes);
pStart += pCol->bytes;
}
taosMemoryFree(keyBuf);
nodesClearList(groupNew);
return code;
}
}
int32_t len = (int32_t)(pStart - (char*)keyBuf);
uint64_t* pGroupId = taosHashGet(pTableListInfo->map, keyBuf, len);
ASSERT(nodeType(pNew) == QUERY_NODE_VALUE);
SValueNode *pValue = (SValueNode *)pNew;
if (!pGroupId) {
uint64_t tmpId = calcGroupId(keyBuf, len);
info->groupId = tmpId;
taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &tmpId, sizeof(uint64_t));
} else {
info->groupId = *pGroupId;
if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL) {
isNull[index++] = 1;
continue;
} else {
isNull[index++] = 0;
char* data = nodesGetValueFromNode(pValue);
if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON){
int32_t len = getJsonValueLen(data);
memcpy(pStart, data, len);
pStart += len;
} else if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) {
memcpy(pStart, data, varDataTLen(data));
pStart += varDataTLen(data);
} else {
memcpy(pStart, data, pValue->node.resType.bytes);
pStart += pValue->node.resType.bytes;
}
}
}
int32_t len = (int32_t)(pStart - (char*)keyBuf);
uint64_t groupId = calcGroupId(keyBuf, len);
taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t));
groupNum++;
nodesClearList(groupNew);
metaReaderClear(&mr);
}
taosMemoryFree(keyBuf);
if(pTableListInfo->needSortTableByGroupId){
return sortTableGroup(pTableListInfo, groupNum);
}
return TDB_CODE_SUCCESS;
}
......@@ -3984,39 +4050,36 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) {
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId);
if (pDataReader == NULL && terrno != 0) {
pTaskInfo->code = terrno;
return NULL;
}
int32_t code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
if (code) {
tsdbCleanupReadHandle(pDataReader);
pTaskInfo->code = terrno;
int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
if(code){
pTaskInfo->code = code;
return NULL;
}
SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags);
code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json
taosArrayDestroy(groupKeys);
code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
if (code) {
tsdbCleanupReadHandle(pDataReader);
pTaskInfo->code = terrno;
return NULL;
}
SOperatorInfo* pOperator = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo);
SOperatorInfo* pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo, queryId, taskId);
STableScanInfo* pScanInfo = pOperator->info;
pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
return pOperator;
} else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
SOperatorInfo* pOperator =
createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
if(code){
return NULL;
}
code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
if (code) {
pTaskInfo->code = terrno;
return NULL;
}
SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
STableScanInfo* pScanInfo = pOperator->info;
pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
return pOperator;
......@@ -4025,46 +4088,17 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table.
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
STimeWindowAggSupp twSup = {
.waterMark = pTableScanNode->watermark,
.calTrigger = pTableScanNode->triggerType,
.maxTs = INT64_MIN,
};
tsdbReaderT pDataReader = NULL;
if (pHandle) {
if (pHandle->initTsdbReader) {
// for stream
ASSERT(pHandle->vnode);
pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId);
} else {
// for tq
ASSERT(pHandle->meta);
getTableList(pHandle->meta, pScanPhyNode, pTableListInfo);
}
}
#if 0
if (pDataReader == NULL && terrno != 0) {
qDebug("%s pDataReader is NULL", GET_TASKID(pTaskInfo));
// return NULL;
} else {
qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo));
createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
}
#endif
SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags);
int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json
taosArrayDestroy(groupKeys);
if (code) {
tsdbCleanupReadHandle(pDataReader);
return NULL;
}
SOperatorInfo* pOperator = createStreamScanOperatorInfo(pDataReader, pHandle, pTableScanNode, pTaskInfo, &twSup);
SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId);
return pOperator;
} else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
......@@ -4093,7 +4127,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
return NULL;
}
} else { // Create one table group.
STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid, .groupId = 0};
STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid};
taosArrayPush(pTableListInfo->pTableList, &info);
}
......@@ -4118,7 +4152,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
cond.suid = pBlockNode->suid;
cond.type = BLOCK_LOAD_OFFSET_SEQ_ORDER;
}
tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId);
tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, queryId, taskId);
cleanupQueryTableDataCond(&cond);
return createDataBlockInfoScanOperator(pReader, pHandle, cond.suid, pBlockNode, pTaskInfo);
......@@ -4355,7 +4389,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle*
goto _error;
}
tsdbReaderT pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId);
tsdbReaderT pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, queryId, taskId);
cleanupQueryTableDataCond(&cond);
return pReader;
......@@ -4584,6 +4618,13 @@ _complete:
static void doDestroyTableList(STableListInfo* pTableqinfoList) {
taosArrayDestroy(pTableqinfoList->pTableList);
taosHashCleanup(pTableqinfoList->map);
if(pTableqinfoList->needSortTableByGroupId){
for(int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++){
SArray* tmp = taosArrayGetP(pTableqinfoList->pGroupList, i);
taosArrayDestroy(tmp);
}
}
taosArrayDestroy(pTableqinfoList->pGroupList);
pTableqinfoList->pTableList = NULL;
pTableqinfoList->map = NULL;
......
source/libs/executor/src/scanoperator.c
浏览文件 @ 086d6f39
......@@ -418,7 +418,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
return NULL;
}
static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
STableScanInfo* pTableScanInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
......@@ -500,6 +500,48 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
}
}
return NULL;
}
static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
STableScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if(pInfo->currentGroupId == -1){
pInfo->currentGroupId++;
if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) {
setTaskStatus(pTaskInfo, TASK_COMPLETED);
return NULL;
}
SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
tsdbCleanupReadHandle(pInfo->dataReader);
tsdbReaderT* pReader = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId);
pInfo->dataReader = pReader;
}
SSDataBlock* result = doTableScanGroup(pOperator);
if(result){
return result;
}
pInfo->currentGroupId++;
if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) {
setTaskStatus(pTaskInfo, TASK_COMPLETED);
return NULL;
}
SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
tsdbSetTableList(pInfo->dataReader, tableList);
tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0);
pInfo->curTWinIdx = 0;
pInfo->scanTimes = 0;
result = doTableScanGroup(pOperator);
if(result){
return result;
}
setTaskStatus(pTaskInfo, TASK_COMPLETED);
return NULL;
}
......@@ -525,8 +567,8 @@ static void destroyTableScanOperatorInfo(void* param, int32_t numOfOutput) {
}
}
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader,
SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) {
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId) {
STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
......@@ -561,10 +603,12 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired;
pInfo->pResBlock = createResDataBlock(pDescNode);
pInfo->pFilterNode = pTableScanNode->scan.node.pConditions;
pInfo->dataReader = pDataReader;
pInfo->scanFlag = MAIN_SCAN;
pInfo->pColMatchInfo = pColList;
pInfo->curTWinIdx = 0;
pInfo->queryId = queryId;
pInfo->taskId = taskId;
pInfo->currentGroupId = -1;
pOperator->name = "TableScanOperator"; // for debug purpose
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN;
......@@ -778,8 +822,9 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info;
pTableScanInfo->cond.twindows[0] = win;
pTableScanInfo->curTWinIdx = 0;
tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
// tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
pTableScanInfo->scanTimes = 0;
pTableScanInfo->currentGroupId = -1;
return true;
}
......@@ -1087,9 +1132,9 @@ static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) {
return tableIdList;
}
SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle,
SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle,
STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo,
STimeWindowAggSupp* pTwSup) {
STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId) {
SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
......@@ -1129,7 +1174,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan
}
if (pHandle) {
SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo);
SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo, queryId, taskId);
STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanDummy->info;
if (pSTInfo->interval.interval > 0) {
pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, pTwSup->waterMark);
......@@ -1889,11 +1934,12 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
goto _error;
}
pInfo->pTableList = pTableListInfo;
pInfo->pColMatchInfo = colList;
pInfo->pRes = createResDataBlock(pDescNode);
pInfo->readHandle = *pReadHandle;
pInfo->curPos = 0;
pInfo->pTableList = pTableListInfo;
pInfo->pColMatchInfo = colList;
pInfo->pRes = createResDataBlock(pDescNode);
pInfo->readHandle = *pReadHandle;
pInfo->curPos = 0;
pOperator->name = "TagScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN;
......@@ -1919,10 +1965,7 @@ _error:
typedef struct STableMergeScanInfo {
STableListInfo* tableListInfo;
int32_t tableStartIndex;
int32_t tableEndIndex;
bool hasGroupId;
uint64_t groupId;
int32_t currentGroupId;
SArray* dataReaders; // array of tsdbReaderT*
SReadHandle readHandle;
......@@ -1968,12 +2011,6 @@ typedef struct STableMergeScanInfo {
SSampleExecInfo sample; // sample execution info
} STableMergeScanInfo;
int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) {
const STableKeyInfo* info1 = p1;
const STableKeyInfo* info2 = p2;
return info1->groupId - info2->groupId;
}
int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) {
int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo);
......@@ -1985,55 +2022,9 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle
qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId);
return TSDB_CODE_SUCCESS;
}
SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags);
generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json
if (groupKeys) {
taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid);
}
taosArrayDestroy(groupKeys);
return TSDB_CODE_SUCCESS;
}
int32_t doCreateMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId,
uint64_t taskId) {
SQueryTableDataCond cond = {0};
int32_t code = initQueryTableDataCond(&cond, pTableScanNode);
code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pPartitionTags);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); ++i) {
STableListInfo* subListInfo = taosMemoryCalloc(1, sizeof(subListInfo));
subListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
taosArrayPush(subListInfo->pTableList, taosArrayGet(pTableListInfo->pTableList, i));
tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, subListInfo, queryId, taskId);
taosArrayPush(arrayReader, &pReader);
taosArrayDestroy(subListInfo->pTableList);
taosMemoryFree(subListInfo);
}
cleanupQueryTableDataCond(&cond);
return TSDB_CODE_SUCCESS;
_error:
return code;
}
int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo,
int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, uint64_t queryId,
uint64_t taskId) {
for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) {
STableListInfo* subListInfo = taosMemoryCalloc(1, sizeof(subListInfo));
subListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
taosArrayPush(subListInfo->pTableList, taosArrayGet(pTableListInfo->pTableList, i));
tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, subListInfo, queryId, taskId);
taosArrayPush(arrayReader, &pReader);
taosArrayDestroy(subListInfo->pTableList);
taosMemoryFree(subListInfo);
return code;
}
return TSDB_CODE_SUCCESS;
......@@ -2225,32 +2216,34 @@ SArray* generateSortByTsInfo(int32_t order) {
return pList;
}
static int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, SArray* tableList, SArray* arrayReader, uint64_t queryId,
uint64_t taskId) {
for (int32_t i = 0; i < taosArrayGetSize(tableList); ++i) {
SArray* tmp = taosArrayInit(1, sizeof(STableKeyInfo));
taosArrayPush(tmp, taosArrayGet(tableList, i));
tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, tmp, queryId, taskId);
taosArrayPush(arrayReader, &pReader);
taosArrayDestroy(tmp);
}
return TSDB_CODE_SUCCESS;
}
int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
STableMergeScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
{
size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
int32_t i = pInfo->tableStartIndex + 1;
for (; i < tableListSize; ++i) {
STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i);
if (tableKeyInfo->groupId != pInfo->groupId) {
break;
}
}
pInfo->tableEndIndex = i - 1;
}
SArray* tableList = taosArrayGetP(pInfo->tableListInfo->pGroupList, pInfo->currentGroupId);
int32_t tableStartIdx = pInfo->tableStartIndex;
int32_t tableEndIdx = pInfo->tableEndIndex;
STableListInfo* tableListInfo = pInfo->tableListInfo;
createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx,
createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableList,
pInfo->dataReaders, pInfo->queryId, pInfo->taskId);
// todo the total available buffer should be determined by total capacity of buffer of this task.
// the additional one is reserved for merge result
pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1);
int32_t tableLen = taosArrayGetSize(tableList);
pInfo->sortBufSize = pInfo->bufPageSize * ((tableLen==0?1:tableLen) + 1);
int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage,
pInfo->pSortInputBlock, pTaskInfo->id.str);
......@@ -2337,38 +2330,43 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
if (code != TSDB_CODE_SUCCESS) {
longjmp(pTaskInfo->env, code);
}
size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
if (!pInfo->hasGroupId) {
pInfo->hasGroupId = true;
if (tableListSize == 0) {
if (pInfo->currentGroupId == -1) {
pInfo->currentGroupId++;
if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) {
doSetOperatorCompleted(pOperator);
return NULL;
}
pInfo->tableStartIndex = 0;
pInfo->groupId = ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId;
startGroupTableMergeScan(pOperator);
}
SSDataBlock* pBlock = NULL;
while (pInfo->tableStartIndex < tableListSize) {
pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator);
if (pBlock != NULL) {
pBlock->info.groupId = pInfo->groupId;
pOperator->resultInfo.totalRows += pBlock->info.rows;
return pBlock;
} else {
stopGroupTableMergeScan(pOperator);
if (pInfo->tableEndIndex >= tableListSize - 1) {
doSetOperatorCompleted(pOperator);
break;
}
pInfo->tableStartIndex = pInfo->tableEndIndex + 1;
pInfo->groupId =
((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId;
startGroupTableMergeScan(pOperator);
}
SSDataBlock* pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator);
if (pBlock != NULL) {
uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t));
if(groupId) pBlock->info.groupId = *groupId;
pOperator->resultInfo.totalRows += pBlock->info.rows;
return pBlock;
}
stopGroupTableMergeScan(pOperator);
pInfo->currentGroupId++;
if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) {
doSetOperatorCompleted(pOperator);
return NULL;
}
startGroupTableMergeScan(pOperator);
pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator);
if (pBlock != NULL) {
uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t));
if(groupId) pBlock->info.groupId = *groupId;
pOperator->resultInfo.totalRows += pBlock->info.rows;
return pBlock;
}
doSetOperatorCompleted(pOperator);
return pBlock;
}
......@@ -2445,6 +2443,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES);
pInfo->queryId = queryId;
pInfo->taskId = taskId;
pInfo->currentGroupId = -1;
pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
......
source/libs/executor/src/timewindowoperator.c
浏览文件 @ 086d6f39
......@@ -3902,6 +3902,7 @@ typedef struct SMergeIntervalAggOperatorInfo {
SIntervalAggOperatorInfo intervalAggOperatorInfo;
SHashObj* groupIntervalHash;
void* groupIntervalIter;
bool hasGroupId;
uint64_t groupId;
SSDataBlock* prefetchedBlock;
......@@ -3914,6 +3915,23 @@ void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) {
destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput);
}
static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win, SSDataBlock* pResultBlock) {
SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo;
SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
bool ascScan = (iaInfo->order == TSDB_ORDER_ASC);
SExprSupp* pExprSup = &pOperatorInfo->exprSupp;
SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &win->skey, TSDB_KEYSIZE, tableGroupId);
SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf,
GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
ASSERT(p1 != NULL);
finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pExprSup->pCtx, pExprSup->pExprInfo,
pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset, pResultBlock, pTaskInfo);
taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
return TSDB_CODE_SUCCESS;
}
static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock,
STimeWindow* newWin) {
SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info;
......@@ -3928,20 +3946,9 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t
return 0;
}
if (newWin == NULL || (ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey)) {
SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &prevWin->skey, TSDB_KEYSIZE, tableGroupId);
SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf,
GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
ASSERT(p1 != NULL);
finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pExprSup->pCtx, pExprSup->pExprInfo,
pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset, pResultBlock, pTaskInfo);
taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE));
if (newWin == NULL) {
taosHashRemove(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId));
} else {
taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow));
}
if ((ascScan && newWin->skey > prevWin->skey || (!ascScan) && newWin->skey < prevWin->skey)) {
finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow));
}
return 0;
......@@ -4090,12 +4097,17 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) {
}
pRes->info.groupId = miaInfo->groupId;
} else {
void* p = taosHashIterate(miaInfo->groupIntervalHash, NULL);
if (p != NULL) {
}
if (miaInfo->inputBlocksFinished) {
void* win = taosHashIterate(miaInfo->groupIntervalHash, miaInfo->groupIntervalIter);
if (win != NULL) {
miaInfo->groupIntervalIter = win;
size_t len = 0;
uint64_t* pKey = taosHashGetKey(p, &len);
outputPrevIntervalResult(pOperator, *pKey, pRes, NULL);
uint64_t* pTableGroupId = taosHashGetKey(win, &len);
finalizeWindowResult(pOperator, *pTableGroupId, win, pRes);
pRes->info.groupId = *pTableGroupId;
}
}
......@@ -4118,6 +4130,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI
}
miaInfo->groupIntervalHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK);
miaInfo->groupIntervalIter = NULL;
SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo;
......
source/libs/nodes/src/nodesCodeFuncs.c
浏览文件 @ 086d6f39
......@@ -2294,6 +2294,7 @@ static const char* jkSubplanType = "SubplanType";
static const char* jkSubplanMsgType = "MsgType";
static const char* jkSubplanLevel = "Level";
static const char* jkSubplanDbFName = "DbFName";
static const char* jkSubplanUser = "User";
static const char* jkSubplanNodeAddr = "NodeAddr";
static const char* jkSubplanRootNode = "RootNode";
static const char* jkSubplanDataSink = "DataSink";
......@@ -2316,6 +2317,9 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkSubplanDbFName, pNode->dbFName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkSubplanUser, pNode->user);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSubplanNodeAddr, queryNodeAddrToJson, &pNode->execNode);
}
......@@ -2352,6 +2356,9 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkSubplanDbFName, pNode->dbFName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkSubplanUser, pNode->user);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkSubplanNodeAddr, jsonToQueryNodeAddr, &pNode->execNode);
}
......
source/libs/nodes/src/nodesUtilFuncs.c
浏览文件 @ 086d6f39
......@@ -1220,6 +1220,7 @@ int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value) {
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_JSON:
pNode->datum.p = (char*)value;
break;
default:
......
source/libs/planner/src/planOptimizer.c
浏览文件 @ 086d6f39
......@@ -1061,6 +1061,7 @@ static EDealRes partTagsOptRebuildTbanmeImpl(SNode** pNode, void* pContext) {
}
strcpy(pFunc->functionName, "tbname");
pFunc->funcType = FUNCTION_TYPE_TBNAME;
pFunc->node.resType = ((SColumnNode*)*pNode)->node.resType;
nodesDestroyNode(*pNode);
*pNode = (SNode*)pFunc;
return DEAL_RES_IGNORE_CHILD;
......@@ -1188,7 +1189,7 @@ static const SOptimizeRule optimizeRuleSet[] = {
{.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize},
{.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize},
{.pName = "SmaIndex", .optimizeFunc = smaOptimize},
// {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize},
{.pName = "PartitionTags", .optimizeFunc = partTagsOptimize},
{.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize}
};
// clang-format on
......
source/libs/planner/src/planPhysiCreater.c
浏览文件 @ 086d6f39
......@@ -1453,6 +1453,9 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl
pSubplan->id = pLogicSubplan->id;
pSubplan->subplanType = pLogicSubplan->subplanType;
pSubplan->level = pLogicSubplan->level;
if (NULL != pCxt->pPlanCxt->pUser) {
strcpy(pSubplan->user, pCxt->pPlanCxt->pUser);
}
return pSubplan;
}
......
source/libs/planner/test/planTestUtil.cpp
浏览文件 @ 086d6f39
......@@ -85,8 +85,9 @@ class PlannerTestBaseImpl {
public:
PlannerTestBaseImpl() : sqlNo_(0) {}
void useDb(const string& acctId, const string& db) {
caseEnv_.acctId_ = acctId;
void useDb(const string& user, const string& db) {
caseEnv_.acctId_ = 0;
caseEnv_.user_ = user;
caseEnv_.db_ = db;
caseEnv_.nsql_ = g_skipSql;
}
......@@ -193,7 +194,8 @@ class PlannerTestBaseImpl {
private:
struct caseEnv {
string acctId_;
int32_t acctId_;
string user_;
string db_;
int32_t nsql_;
......@@ -295,7 +297,7 @@ class PlannerTestBaseImpl {
transform(stmtEnv_.sql_.begin(), stmtEnv_.sql_.end(), stmtEnv_.sql_.begin(), ::tolower);
SParseContext cxt = {0};
cxt.acctId = atoi(caseEnv_.acctId_.c_str());
cxt.acctId = caseEnv_.acctId_;
cxt.db = caseEnv_.db_.c_str();
cxt.pSql = stmtEnv_.sql_.c_str();
cxt.sqlLen = stmtEnv_.sql_.length();
......@@ -319,12 +321,13 @@ class PlannerTestBaseImpl {
void doParseBoundSql(SQuery* pQuery) {
SParseContext cxt = {0};
cxt.acctId = atoi(caseEnv_.acctId_.c_str());
cxt.acctId = caseEnv_.acctId_;
cxt.db = caseEnv_.db_.c_str();
cxt.pSql = stmtEnv_.sql_.c_str();
cxt.sqlLen = stmtEnv_.sql_.length();
cxt.pMsg = stmtEnv_.msgBuf_.data();
cxt.msgLen = stmtEnv_.msgBuf_.max_size();
cxt.pUser = caseEnv_.user_.c_str();
DO_WITH_THROW(qStmtParseQuerySql, &cxt, pQuery);
res_.ast_ = toString(pQuery->pRoot);
......@@ -364,6 +367,7 @@ class PlannerTestBaseImpl {
void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) {
pCxt->queryId = 1;
pCxt->pUser = caseEnv_.user_.c_str();
if (QUERY_NODE_CREATE_TOPIC_STMT == nodeType(pQuery->pRoot)) {
pCxt->pAstRoot = ((SCreateTopicStmt*)pQuery->pRoot)->pQuery;
pCxt->topicQuery = true;
......@@ -403,7 +407,7 @@ PlannerTestBase::PlannerTestBase() : impl_(new PlannerTestBaseImpl()) {}
PlannerTestBase::~PlannerTestBase() {}
void PlannerTestBase::useDb(const std::string& acctId, const std::string& db) { impl_->useDb(acctId, db); }
void PlannerTestBase::useDb(const std::string& user, const std::string& db) { impl_->useDb(user, db); }
void PlannerTestBase::run(const std::string& sql) { return impl_->run(sql); }
......
source/libs/planner/test/planTestUtil.h
浏览文件 @ 086d6f39
......@@ -30,7 +30,7 @@ class PlannerTestBase : public testing::Test {
PlannerTestBase();
virtual ~PlannerTestBase();
void useDb(const std::string& acctId, const std::string& db);
void useDb(const std::string& user, const std::string& db);
void run(const std::string& sql);
// stmt mode APIs
void prepare(const std::string& sql);
......
source/libs/stream/src/streamDispatch.c
浏览文件 @ 086d6f39
......@@ -63,7 +63,6 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) {
}
int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq) {
//
if (tStartEncode(pEncoder) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->dstNodeId) < 0) return -1;
......@@ -84,7 +83,7 @@ int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) {
if (tDecodeI32(pDecoder, &pReq->srcTaskId) < 0) return -1;
uint64_t len = 0;
if (tDecodeBinaryAlloc(pDecoder, (void**)&pReq->pRetrieve, &len) < 0) return -1;
pReq->retrieveLen = len;
pReq->retrieveLen = (int32_t)len;
tEndDecode(pDecoder);
return 0;
}
......
source/libs/sync/src/syncElection.c
浏览文件 @ 086d6f39
......@@ -105,9 +105,16 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) {
}
int32_t syncNodeRequestVote(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncRequestVote* pMsg) {
sTrace("syncNodeRequestVote pSyncNode:%p ", pSyncNode);
int32_t ret = 0;
do {
char host[128];
uint16_t port;
syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port);
sDebug("vgId:%d, send sync-request-vote to %s:%d, {term:%lu, last-index:%ld, last-term:%lu}", pSyncNode->vgId, host,
port, pMsg->term, pMsg->lastLogTerm, pMsg->lastLogIndex);
} while (0);
SRpcMsg rpcMsg;
syncRequestVote2RpcMsg(pMsg, &rpcMsg);
syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg);
......
source/libs/sync/src/syncMain.c
浏览文件 @ 086d6f39
......@@ -370,6 +370,15 @@ bool syncIsReady(int64_t rid) {
ASSERT(rid == pSyncNode->rid);
bool b = (pSyncNode->state == TAOS_SYNC_STATE_LEADER) && pSyncNode->restoreFinish;
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
// if false, set error code
if (false == b) {
if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) {
terrno = TSDB_CODE_SYN_NOT_LEADER;
} else {
terrno = TSDB_CODE_APP_NOT_READY;
}
}
return b;
}
......@@ -480,12 +489,30 @@ void syncGetEpSet(int64_t rid, SEpSet* pEpSet) {
snprintf(pEpSet->eps[i].fqdn, sizeof(pEpSet->eps[i].fqdn), "%s", (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodeFqdn);
pEpSet->eps[i].port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort;
(pEpSet->numOfEps)++;
sInfo("syncGetEpSet index:%d %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
sInfo("vgId:%d sync get epset: index:%d %s:%d", pSyncNode->vgId, i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
}
pEpSet->inUse = pSyncNode->pRaftCfg->cfg.myIndex;
sInfo("vgId:%d sync get epset in-use:%d", pSyncNode->vgId, pEpSet->inUse);
sInfo("syncGetEpSet pEpSet->inUse:%d ", pEpSet->inUse);
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
}
void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) {
SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
if (pSyncNode == NULL) {
memset(pEpSet, 0, sizeof(*pEpSet));
return;
}
ASSERT(rid == pSyncNode->rid);
pEpSet->numOfEps = 0;
for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
snprintf(pEpSet->eps[i].fqdn, sizeof(pEpSet->eps[i].fqdn), "%s", (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodeFqdn);
pEpSet->eps[i].port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort;
(pEpSet->numOfEps)++;
sInfo("vgId:%d sync get retry epset: index:%d %s:%d", pSyncNode->vgId, i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
}
pEpSet->inUse = (pSyncNode->pRaftCfg->cfg.myIndex + 1) % pEpSet->numOfEps;
sInfo("vgId:%d sync get retry epset in-use:%d", pSyncNode->vgId, pEpSet->inUse);
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
}
......
source/libs/sync/src/syncReplication.c
浏览文件 @ 086d6f39
......@@ -224,8 +224,8 @@ int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, c
uint16_t port;
syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port);
sDebug(
"vgId:%d, send sync-append-entries to %s:%d, term:%lu, pre-index:%ld, pre-term:%lu, pterm:%lu, commit:%ld, "
"datalen:%d",
"vgId:%d, send sync-append-entries to %s:%d, {term:%lu, pre-index:%ld, pre-term:%lu, pterm:%lu, commit:%ld, "
"datalen:%d}",
pSyncNode->vgId, host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm,
pMsg->commitIndex, pMsg->dataLen);
} while (0);
......
source/libs/sync/src/syncRequestVote.c
浏览文件 @ 086d6f39
......@@ -99,15 +99,20 @@ static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pM
int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) {
int32_t ret = 0;
// print log
char logBuf[128] = {0};
snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVote, currentTerm:%lu", ths->pRaftStore->currentTerm);
syncRequestVoteLog2(logBuf, pMsg);
// if already drop replica, do not process
if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) {
sInfo("recv SyncRequestVote maybe replica already dropped");
return ret;
do {
char logBuf[256];
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
snprintf(logBuf, sizeof(logBuf),
"recv sync-request-vote from %s:%d, term:%lu, lindex:%ld, lterm:%lu, maybe replica already dropped",
host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm);
syncNodeEventLog(ths, logBuf);
} while (0);
return -1;
}
// maybe update term
......@@ -135,10 +140,22 @@ int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) {
pReply->term = ths->pRaftStore->currentTerm;
pReply->voteGranted = grant;
// trace log
do {
char logBuf[256];
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
snprintf(logBuf, sizeof(logBuf),
"recv sync-request-vote from %s:%d, term:%lu, lindex:%ld, lterm:%lu, reply-grant:%d", host, port,
pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, pReply->voteGranted);
syncNodeEventLog(ths, logBuf);
} while (0);
SRpcMsg rpcMsg;
syncRequestVoteReply2RpcMsg(pReply, &rpcMsg);
syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg);
syncRequestVoteReplyDestroy(pReply);
return ret;
return 0;
}
\ No newline at end of file
tests/parallel_test/run.sh
浏览文件 @ 086d6f39
......@@ -8,11 +8,12 @@ function usage() {
echo -e "\t -l log dir"
echo -e "\t -e enterprise edition"
echo -e "\t -o default timeout value"
echo -e "\t -w log web server"
echo -e "\t -h help"
}
ent=0
while getopts "m:t:b:l:o:eh" opt; do
while getopts "m:t:b:l:o:w:eh" opt; do
case $opt in
m)
config_file=$OPTARG
......@@ -32,6 +33,9 @@ while getopts "m:t:b:l:o:eh" opt; do
o)
timeout_param="-o $OPTARG"
;;
w)
web_server=$OPTARG
;;
h)
usage
exit 0
......@@ -64,10 +68,11 @@ if [ ! -f $t_file ]; then
exit 1
fi
date_tag=`date +%Y%m%d-%H%M%S`
test_log_dir=${branch}_${date_tag}
if [ -z $log_dir ]; then
log_dir="log/${branch}_${date_tag}"
log_dir="log/${test_log_dir}"
else
log_dir="$log_dir/${branch}_${date_tag}"
log_dir="$log_dir/${test_log_dir}"
fi
hosts=()
......@@ -190,44 +195,54 @@ function run_thread() {
# echo "$thread_no $count $cmd"
local ret=0
local redo_count=1
local case_log_file=$log_dir/${case_file}.txt
start_time=`date +%s`
local case_index=`flock -x $lock_file -c "sh -c \"echo \\\$(( \\\$( cat $index_file ) + 1 )) | tee $index_file\""`
case_index=`printf "%5d" $case_index`
local case_info=`echo "$line"|cut -d, -f 3,4`
while [ ${redo_count} -lt 6 ]; do
if [ -f $log_dir/$case_file.log ]; then
cp $log_dir/$case_file.log $log_dir/$case_file.${redo_count}.redolog
if [ -f $case_log_file ]; then
cp $case_log_file $log_dir/$case_file.${redo_count}.redotxt
fi
echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >$log_dir/$case_file.log
echo -e "\e[33m >>>>> \e[0m ${case_cmd}"
date >>$log_dir/$case_file.log
# $cmd 2>&1 | tee -a $log_dir/$case_file.log
echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >$case_log_file
local current_time=`date "+%Y-%m-%d %H:%M:%S"`
echo -e "$case_index \e[33m START >>>>> \e[0m ${case_info} \e[33m[$current_time]\e[0m"
echo "$current_time" >>$case_log_file
local real_start_time=`date +%s`
# $cmd 2>&1 | tee -a $case_log_file
# ret=${PIPESTATUS[0]}
$cmd >>$log_dir/$case_file.log 2>&1
$cmd >>$case_log_file 2>&1
ret=$?
echo "${hosts[index]} `date` ret:${ret}" >>$log_dir/$case_file.log
local real_end_time=`date +%s`
local time_elapsed=$(( real_end_time - real_start_time ))
echo "execute time: ${time_elapsed}s" >>$case_log_file
current_time=`date "+%Y-%m-%d %H:%M:%S"`
echo "${hosts[index]} $current_time exit code:${ret}" >>$case_log_file
if [ $ret -eq 0 ]; then
break
fi
redo=0
grep -q "wait too long for taosd start" $log_dir/$case_file.log
grep -q "wait too long for taosd start" $case_log_file
if [ $? -eq 0 ]; then
redo=1
fi
grep -q "kex_exchange_identification: Connection closed by remote host" $log_dir/$case_file.log
grep -q "kex_exchange_identification: Connection closed by remote host" $case_log_file
if [ $? -eq 0 ]; then
redo=1
fi
grep -q "ssh_exchange_identification: Connection closed by remote host" $log_dir/$case_file.log
grep -q "ssh_exchange_identification: Connection closed by remote host" $case_log_file
if [ $? -eq 0 ]; then
redo=1
fi
grep -q "kex_exchange_identification: read: Connection reset by peer" $log_dir/$case_file.log
grep -q "kex_exchange_identification: read: Connection reset by peer" $case_log_file
if [ $? -eq 0 ]; then
redo=1
fi
grep -q "Database not ready" $log_dir/$case_file.log
grep -q "Database not ready" $case_log_file
if [ $? -eq 0 ]; then
redo=1
fi
grep -q "Unable to establish connection" $log_dir/$case_file.log
grep -q "Unable to establish connection" $case_log_file
if [ $? -eq 0 ]; then
redo=1
fi
......@@ -240,11 +255,18 @@ function run_thread() {
redo_count=$(( redo_count + 1 ))
done
end_time=`date +%s`
echo >>$log_dir/$case_file.log
echo "${hosts[index]} execute time: $(( end_time - start_time ))s" >>$log_dir/$case_file.log
echo >>$case_log_file
total_time=$(( end_time - start_time ))
echo "${hosts[index]} total time: ${total_time}s" >>$case_log_file
# echo "$thread_no ${line} DONE"
if [ $ret -ne 0 ]; then
flock -x $lock_file -c "echo \"${hosts[index]} ret:${ret} ${line}\" >>$log_dir/failed.log"
if [ $ret -eq 0 ]; then
echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[32m success\e[0m"
else
if [ ! -z ${web_server} ]; then
flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n ${web_server}/$test_log_dir/${case_file}.txt\" >>${failed_case_file}"
else
flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n log file: ${case_log_file}\" >>${failed_case_file}"
fi
mkdir -p $log_dir/${case_file}.coredump
local remote_coredump_dir="${workdirs[index]}/tmp/thread_volume/$thread_no/coredump"
local scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}"
......@@ -253,13 +275,15 @@ function run_thread() {
fi
cmd="$scpcmd:${remote_coredump_dir}/* $log_dir/${case_file}.coredump/"
$cmd # 2>/dev/null
local case_info=`echo "$line"|cut -d, -f 3,4`
local corefile=`ls $log_dir/${case_file}.coredump/`
echo -e "$case_info \e[31m failed\e[0m"
echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[31m failed\e[0m"
echo "=========================log============================"
cat $log_dir/$case_file.log
cat $case_log_file
echo "====================================================="
echo -e "\e[34m log file: $log_dir/$case_file.log \e[0m"
echo -e "\e[34m log file: $case_log_file \e[0m"
if [ ! -z "${web_server}" ]; then
echo "${web_server}/$test_log_dir/${case_file}.txt"
fi
if [ ! -z "$corefile" ]; then
echo -e "\e[34m corefiles: $corefile \e[0m"
local build_dir=$log_dir/build_${hosts[index]}
......@@ -325,6 +349,10 @@ mkdir -p $log_dir
rm -rf $log_dir/*
task_file=$log_dir/$$.task
lock_file=$log_dir/$$.lock
index_file=$log_dir/case_index.txt
stat_file=$log_dir/stat.txt
failed_case_file=$log_dir/failed.txt
echo "0" >$index_file
i=0
j=0
......@@ -350,15 +378,45 @@ rm -f $lock_file
rm -f $task_file
# docker ps -a|grep -v CONTAINER|awk '{print $1}'|xargs docker rm -f
echo "====================================================================="
echo "log dir: $log_dir"
total_cases=`cat $index_file`
failed_cases=0
if [ -f $failed_case_file ]; then
if [ ! -z "$web_server" ]; then
failed_cases=`grep -v "$web_server" $failed_case_file|wc -l`
else
failed_cases=`grep -v "log file:" $failed_case_file|wc -l`
fi
fi
success_cases=$(( total_cases - failed_cases ))
echo "Total Cases: $total_cases" >$stat_file
echo "Successful: $success_cases" >>$stat_file
echo "Failed: $failed_cases" >>$stat_file
cat $stat_file
RET=0
i=1
if [ -f "$log_dir/failed.log" ]; then
if [ -f "${failed_case_file}" ]; then
echo "====================================================="
while read line; do
if [ ! -z "${web_server}" ]; then
echo "$line"|grep -q "${web_server}"
if [ $? -eq 0 ]; then
echo " $line"
continue
fi
else
echo "$line"|grep -q "log file:"
if [ $? -eq 0 ]; then
echo " $line"
continue
fi
fi
line=`echo "$line"|cut -d, -f 3,4`
echo -e "$i. $line \e[31m failed\e[0m" >&2
i=$(( i + 1 ))
done <$log_dir/failed.log
done <${failed_case_file}
RET=1
fi
......
tests/script/jenkins/basic.txt
浏览文件 @ 086d6f39
......@@ -2,12 +2,10 @@
#======================b1-start===============
# ---- user
./test.sh -f tsim/user/basic1.sim
./test.sh -f tsim/user/pass_alter.sim
./test.sh -f tsim/user/pass_len.sim
./test.sh -f tsim/user/user_len.sim
./test.sh -f tsim/user/privilege1.sim
./test.sh -f tsim/user/privilege2.sim
./test.sh -f tsim/user/basic.sim
./test.sh -f tsim/user/password.sim
./test.sh -f tsim/user/privilege_db.sim
./test.sh -f tsim/user/privilege_sysinfo.sim
## ---- db
./test.sh -f tsim/db/create_all_options.sim
......@@ -135,7 +133,7 @@
./test.sh -f tsim/stable/tag_filter.sim
# --- for multi process mode
./test.sh -f tsim/user/basic1.sim -m
./test.sh -f tsim/user/basic.sim -m
./test.sh -f tsim/db/basic3.sim -m
./test.sh -f tsim/db/error1.sim -m
./test.sh -f tsim/insert/backquote.sim -m
......
tests/script/tsim/mnode/basic5.sim
浏览文件 @ 086d6f39
......@@ -157,7 +157,7 @@ step61:
if $x == 10 then
return -1
endi
sql show mnodes
sql show mnodes -x step61
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
......
tests/script/tsim/user/basic.sim 0 → 100644
浏览文件 @ 086d6f39
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
print =============== step0
sql show users
if $data(root)[1] != 1 then
return -1
endi
if $data(root)[2] != 1 then
return -1
endi
if $data(root)[3] != 1 then
return -1
endi
sql alter user root pass 'taosdata'
sql_error ALTER USER root SYSINFO 0
sql_error ALTER USER root SYSINFO 1
sql_error ALTER USER root enable 0
sql_error ALTER USER root enable 1
sql_error create database db vgroups 1;
sql_error GRANT read ON db.* to root;
sql_error GRANT read ON *.* to root;
sql_error REVOKE read ON db.* from root;
sql_error REVOKE read ON *.* from root;
sql_error GRANT write ON db.* to root;
sql_error GRANT write ON *.* to root;
sql_error REVOKE write ON db.* from root;
sql_error REVOKE write ON *.* from root;
sql_error REVOKE write ON *.* from root;
sql_error GRANT all ON *.* to root;
sql_error REVOKE all ON *.* from root;
sql_error GRANT read,write ON *.* to root;
sql_error REVOKE read,write ON *.* from root;
print =============== step1: sysinfo create
sql CREATE USER u1 PASS 'taosdata' SYSINFO 0;
sql show users
if $rows != 2 then
return -1
endi
if $data(u1)[1] != 0 then
return -1
endi
if $data(u1)[2] != 1 then
return -1
endi
if $data(u1)[3] != 0 then
return -1
endi
sql CREATE USER u2 PASS 'taosdata' SYSINFO 1;
sql show users
if $rows != 3 then
return -1
endi
if $data(u2)[1] != 0 then
return -1
endi
if $data(u2)[2] != 1 then
return -1
endi
if $data(u2)[3] != 1 then
return -1
endi
print =============== step2: sysinfo alter
sql ALTER USER u1 SYSINFO 1
sql show users
if $data(u1)[1] != 0 then
return -1
endi
if $data(u1)[2] != 1 then
return -1
endi
if $data(u1)[3] != 1 then
return -1
endi
sql ALTER USER u1 SYSINFO 0
sql show users
if $data(u1)[1] != 0 then
return -1
endi
if $data(u1)[2] != 1 then
return -1
endi
if $data(u1)[3] != 0 then
return -1
endi
sql ALTER USER u1 SYSINFO 0
sql ALTER USER u1 SYSINFO 0
sql drop user u1
sql show users
if $rows != 2 then
return -1
endi
print =============== step3: enable alter
sql ALTER USER u2 enable 0
sql show users
if $rows != 2 then
return -1
endi
if $data(u2)[1] != 0 then
return -1
endi
if $data(u2)[2] != 0 then
return -1
endi
if $data(u2)[3] != 1 then
return -1
endi
sql ALTER USER u2 enable 1
sql show users
if $data(u2)[1] != 0 then
return -1
endi
if $data(u2)[2] != 1 then
return -1
endi
if $data(u2)[3] != 1 then
return -1
endi
sql ALTER USER u2 enable 1
sql ALTER USER u2 enable 1
print =============== restart taosd
system sh/exec.sh -n dnode1 -s stop
system sh/exec.sh -n dnode1 -s start
print =============== step4: enable privilege
sql show users
if $rows != 2 then
return -1
endi
if $data(u2)[1] != 0 then
return -1
endi
if $data(u2)[2] != 1 then
return -1
endi
if $data(u2)[3] != 1 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
tests/script/tsim/user/basic1.sim 已删除 100644 → 0
浏览文件 @ 37d1cc28
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
print =============== show users
sql show users
if $rows != 1 then
return -1
endi
print $data[0][0] $data[0][1] $data[0][2]
print $data[1][0] $data[1][1] $data[1][2]
print $data[2][0] $data[1][2] $data[2][2]
sql_error show accounts;
sql_error create account a pass "a"
sql_error drop account a
sql_error drop account root
print =============== create user1
sql create user user1 PASS 'user1'
sql show users
if $rows != 2 then
return -1
endi
print $data[0][0] $data[0][1] $data[0][2]
print $data[1][0] $data[1][1] $data[1][2]
print $data[2][0] $data[1][2] $data[2][2]
print $data[3][0] $data[3][1] $data[3][2]
print =============== create user2
sql create user user2 PASS 'user2'
sql show users
if $rows != 3 then
return -1
endi
print $data[0][0] $data[0][1] $data[0][2]
print $data[1][0] $data[1][1] $data[1][2]
print $data[2][0] $data[1][2] $data[2][2]
print $data[3][0] $data[3][1] $data[3][2]
print $data40 $data41 $data42
print =============== drop user1
sql drop user user1
sql show users
if $rows != 2 then
return -1
endi
print $data[0][0] $data[0][1] $data[0][2]
print $data[1][0] $data[1][1] $data[1][2]
print $data[2][0] $data[1][2] $data[2][2]
print $data[3][0] $data[3][1] $data[3][2]
print =============== restart taosd
system sh/exec.sh -n dnode1 -s stop
sleep 1000
system sh/exec.sh -n dnode1 -s start
print =============== show users
sql show users
if $rows != 2 then
return -1
endi
print $data[0][0] $data[0][1] $data[0][2]
print $data[1][0] $data[1][1] $data[1][2]
print $data[2][0] $data[1][2] $data[2][2]
print $data[3][0] $data[3][1] $data[3][2]
system sh/exec.sh -n dnode1 -s stop -x SIGINT
tests/script/tsim/user/pass_len.sim 已删除 100644 → 0
浏览文件 @ 37d1cc28
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
$i = 0
$dbPrefix = apdb
$tbPrefix = aptb
$db = $dbPrefix . $i
$tb = $tbPrefix . $i
$userPrefix = apusr
print =============== step1
$i = 0
$user = $userPrefix . $i
sql drop user $user -x step11
return -1
step11:
sql create user $user PASS -x step12
return -1
step12:
sql create user $user PASS 'taosdata'
sql show users
if $rows != 2 then
return -1
endi
print =============== step2
$i = 1
$user = $userPrefix . $i
sql drop user $user -x step2
step2:
sql create user $user PASS '1'
sql show users
if $rows != 3 then
return -1
endi
print =============== step3
$i = 2
$user = $userPrefix . $i
sql drop user $user -x step3
step3:
sql create user $user PASS 'abc0123456789'
sql show users
if $rows != 4 then
return -1
endi
print =============== step4
$i = 3
$user = $userPrefix . $i
sql create user $user PASS 'abcd012345678901234567891234567890abcd012345678901234567891234567890abcd012345678901234567891234567890abcd012345678901234567891234567890123' -x step4
return -1
step4:
sql show users
if $rows != 4 then
return -1
endi
$i = 0
while $i < 3
$user = $userPrefix . $i
sql drop user $user
$i = $i + 1
endw
sql show users
if $rows != 1 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
tests/script/tsim/user/pass_alter.sim → tests/script/tsim/user/password.sim
浏览文件 @ 086d6f39
......@@ -16,14 +16,12 @@ if $rows != 3 then
endi
print ============= step2
sql close
sleep 2500
print user u_read login
sql close
sql connect u_read
sql alter user u_read pass 'taosdata'
sql alter user u_write pass 'taosdata1' -x step2
return -1
step2:
sql_error alter user u_write pass 'taosdata1'
sql_error create user read1 pass 'taosdata1'
sql_error create user write1 pass 'taosdata1'
......@@ -34,17 +32,14 @@ if $rows != 3 then
endi
print ============= step3
sql close
sleep 2500
print user u_write login
sql close
sql connect u_write
sql_error create user read2 pass 'taosdata1'
sql_error create user write2 pass 'taosdata1'
sql alter user u_write pass 'taosdata'
sql alter user u_read pass 'taosdata' -x step3
return -1
step3:
sql_error alter user u_read pass 'taosdata'
sql show users
if $rows != 3 then
......@@ -52,15 +47,41 @@ if $rows != 3 then
endi
print ============= step4
sql close
sleep 2500
print user root login
sql close
sql connect
sql create user oroot pass 'taosdata'
sql_error create user $user PASS 'abcd012345678901234567891234567890abcd012345678901234567891234567890abcd012345678901234567891234567890abcd012345678901234567891234567890123'
sql_error create userabcd012345678901234567891234567890abcd01234567890123456789123456789 PASS 'taosdata'
sql_error create user abcd0123456789012345678901234567890111 PASS '123'
sql create user abc01234567890123456789 PASS '123'
sql show users
if $rows != 4 then
if $rows != 5 then
return -1
endi
print ============= step5
sql create database db vgroups 1
sql_error ALTER USER o_root SYSINFO 0
sql_error ALTER USER o_root SYSINFO 1
sql_error ALTER USER o_root enable 0
sql_error ALTER USER o_root enable 1
sql_error create database db vgroups 1;
sql_error GRANT read ON db.* to o_root;
sql_error GRANT read ON *.* to o_root;
sql_error REVOKE read ON db.* from o_root;
sql_error REVOKE read ON *.* from o_root;
sql_error GRANT write ON db.* to o_root;
sql_error GRANT write ON *.* to o_root;
sql_error REVOKE write ON db.* from o_root;
sql_error REVOKE write ON *.* from o_root;
sql_error REVOKE write ON *.* from o_root;
sql_error GRANT all ON *.* to o_root;
sql_error REVOKE all ON *.* from o_root;
sql_error GRANT read,write ON *.* to o_root;
sql_error REVOKE read,write ON *.* from o_root;
system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
tests/script/tsim/user/privilege2.sim 已删除 100644 → 0
浏览文件 @ 37d1cc28
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
print =============== show users
sql create database d1 vgroups 1;
sql create database d2 vgroups 1;
sql create database d3 vgroups 1;
sql show databases
if $rows != 5 then
return -1
endi
print =============== create users
sql create user user1 PASS 'taosdata'
sql create user user2 PASS 'taosdata'
sql show users
if $rows != 3 then
return -1
endi
sql GRANT read ON d1.* to user1;
sql GRANT write ON d2.* to user1;
print =============== re connect
sql close
sleep 2500
print user user1 login
sql connect user1
sql_error drop database d1;
sql_error drop database d2;
sql_error create stable d1.st (ts timestamp, i int) tags (j int)
sql create stable d2.st (ts timestamp, i int) tags (j int)
system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
tests/script/tsim/user/privilege1.sim → tests/script/tsim/user/privilege_db.sim
浏览文件 @ 086d6f39
......@@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
print =============== show users
print =============== create db
sql create database d1 vgroups 1;
sql create database d2 vgroups 1;
sql create database d3 vgroups 1;
......@@ -68,4 +68,26 @@ sql REVOKE read,write ON d1.* from user1;
sql REVOKE read,write ON d2.* from user1;
sql REVOKE read,write ON *.* from user1;
print =============== create users
sql create user u1 PASS 'taosdata'
sql show users
if $rows != 4 then
return -1
endi
sql GRANT read ON d1.* to u1;
sql GRANT write ON d2.* to u1;
print =============== re connect
print user u1 login
sql close
sql connect u1
sql_error drop database d1;
sql_error drop database d2;
sql_error create stable d1.st (ts timestamp, i int) tags (j int)
sql create stable d2.st (ts timestamp, i int) tags (j int)
system sh/exec.sh -n dnode1 -s stop -x SIGINT
tests/script/tsim/user/privilege_sysinfo.sim 0 → 100644
浏览文件 @ 086d6f39
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
print =============== create user and login
sql create user sysinfo0 pass 'taosdata'
sql create user sysinfo1 pass 'taosdata'
sql alter user sysinfo0 sysinfo 0
sql alter user sysinfo1 sysinfo 1
print user sysinfo0 login
sql close
sql connect sysinfo0
system sh/exec.sh -n dnode1 -s stop
return
print =============== check oper
sql_error create user u1 pass 'u1'
sql_error drop user sysinfo1
sql_error alter user sysinfo1 pass '1'
sql_error alter user sysinfo0 pass '1'
system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
tests/script/tsim/user/user_len.sim 已删除 100644 → 0
浏览文件 @ 37d1cc28
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
$i = 0
$dbPrefix = lm_us_db
$tbPrefix = lm_us_tb
$db = $dbPrefix . $i
$tb = $tbPrefix . $i
print =============== step1
sql drop user ac -x step0
return -1
step0:
sql create user PASS '123' -x step1
return -1
step1:
sql show users
if $rows != 1 then
return -1
endi
print =============== step2
sql drop user a -x step2
step2:
sql create user a PASS '123'
sql show users
if $rows != 2 then
return -1
endi
sql drop user a
sql show users
if $rows != 1 then
return -1
endi
print =============== step3
sql drop user abc01234567890123456789 -x step3
step3:
sql create user abc01234567890123456789 PASS '123'
sql show users
if $rows != 2 then
return -1
endi
sql drop user abc01234567890123456789
sql show users
if $rows != 1 then
return -1
endi
print =============== step4
sql create user abcd0123456789012345678901234567890111 PASS '123' -x step4
return -1
step4:
sql show users
if $rows != 1 then
return -1
endi
print =============== step5
sql drop user 123 -x step5
step5:
sql create user 123 PASS '123' -x step61
return -1
step61:
sql create user a123 PASS '123'
sql show users
if $rows != 2 then
return -1
endi
sql drop user a123
sql show users
if $rows != 1 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
tests/system-test/2-query/function_null.py 0 → 100644
浏览文件 @ 086d6f39
import taos
import sys
import datetime
import inspect
from util.log import *
from util.sql import *
from util.cases import *
import random
class TDTestCase:
updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143,
"jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143,
"wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143}
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
self.tb_nums = 10
self.row_nums = 20
self.ts = 1434938400000
self.time_step = 1000
def prepare_tag_datas(self):
# prepare datas
tdSql.execute(
"create database if not exists testdb keep 3650 duration 1000")
tdSql.execute(" use testdb ")
tdSql.execute(
'''create table stb1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32))
'''
)
tdSql.execute(
'''
create table t1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
'''
)
for i in range(4):
tdSql.execute(
f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )')
for i in range(9):
tdSql.execute(
f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
"insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
tdSql.execute(
"insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
"insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
"insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
f'''insert into t1 values
( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
'''
)
def function_for_null_data(self):
function_names = ["abs" , "floor" , "ceil" , "round"]
for function_name in function_names:
scalar_sql_1 = f"select {function_name}(c1)/0 from t1 group by c1 order by c1"
scalar_sql_2 = f"select {function_name}(c1/0) from t1 group by c1 order by c1"
scalar_sql_3 = f"select {function_name}(NULL) from t1 group by c1 order by c1"
tdSql.query(scalar_sql_1)
tdSql.checkRows(10)
tdSql.checkData(0,0,None)
tdSql.checkData(9,0,None)
tdSql.query(scalar_sql_2)
tdSql.checkRows(10)
tdSql.checkData(0,0,None)
tdSql.checkData(9,0,None)
tdSql.query(scalar_sql_3)
tdSql.checkRows(10)
tdSql.checkData(0,0,None)
tdSql.checkData(9,0,None)
function_names = ["sin" ,"cos" ,"tan" ,"asin" ,"acos" ,"atan"]
PI = 3.141592654
# sin
tdSql.query(" select sin(c1/0) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select sin(NULL) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select sin(0.00) from t1 group by c1 order by c1")
tdSql.checkData(9,0,0.000000000)
tdSql.query(f" select sin({PI/2}) from t1 group by c1 order by c1")
tdSql.checkData(9,0,1.0)
tdSql.query(" select sin(1000) from t1 group by c1 order by c1")
tdSql.checkData(9,0,0.826879541)
# cos
tdSql.query(" select cos(c1/0) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select cos(NULL) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select cos(0.00) from t1 group by c1 order by c1")
tdSql.checkData(9,0,1.000000000)
tdSql.query(f" select cos({PI}/2) from t1 group by c1 order by c1")
tdSql.query(" select cos(1000) from t1 group by c1 order by c1")
tdSql.checkData(9,0,0.562379076)
# tan
tdSql.query(" select tan(c1/0) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select tan(NULL) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select tan(0.00) from t1 group by c1 order by c1")
tdSql.checkData(9,0,0.000000000)
tdSql.query(f" select tan({PI}/2) from t1 group by c1 order by c1")
tdSql.query(" select tan(1000) from t1 group by c1 order by c1")
tdSql.checkData(9,0,1.470324156)
# atan
tdSql.query(" select atan(c1/0) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select atan(NULL) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select atan(0.00) from t1 group by c1 order by c1")
tdSql.checkData(9,0,0.000000000)
tdSql.query(f" select atan({PI}/2) from t1 group by c1 order by c1")
tdSql.checkData(9,0,1.003884822)
tdSql.query(" select atan(1000) from t1 group by c1 order by c1")
tdSql.checkData(9,0,1.569796327)
# asin
tdSql.query(" select asin(c1/0) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select asin(NULL) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select asin(0.00) from t1 group by c1 order by c1")
tdSql.checkData(9,0,0.000000000)
tdSql.query(f" select asin({PI}/2) from t1 group by c1 order by c1")
tdSql.query(" select asin(1000) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
# acos
tdSql.query(" select acos(c1/0) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select acos(NULL) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select acos(0.00) from t1 group by c1 order by c1")
tdSql.checkData(9,0,1.570796327)
tdSql.query(f" select acos({PI}/2) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select acos(1000) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
function_names = ["log" ,"pow"]
# log
tdSql.query(" select log(-10) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select log(NULL ,2) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select log(c1)/0 from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(f" select log(0.00) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
# pow
tdSql.query(" select pow(c1,10000) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select pow(c1,2)/0 from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(" select pow(NULL,2) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
tdSql.query(f" select pow(c1/0 ,1 ) from t1 group by c1 order by c1")
tdSql.checkData(9,0,None)
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
tdLog.printNoPrefix("==========step1:create table ==============")
self.prepare_tag_datas()
tdLog.printNoPrefix("==========step2:test errors ==============")
self.function_for_null_data()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())
\ No newline at end of file
tests/system-test/2-query/irate.py 0 → 100644
浏览文件 @ 086d6f39
import taos
import sys
import datetime
import inspect
from util.log import *
from util.sql import *
from util.cases import *
import random
class TDTestCase:
updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143,
"jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143,
"wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143}
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
self.tb_nums = 10
self.row_nums = 20
self.ts = 1434938400000
self.time_step = 1000
def insert_datas_and_check_irate(self ,tbnums , rownums , time_step ):
tdLog.info(" prepare datas for auto check irate function ")
tdSql.execute(" create database test ")
tdSql.execute(" use test ")
tdSql.execute(" create stable stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\
c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)")
for tbnum in range(tbnums):
tbname = "sub_tb_%d"%tbnum
tdSql.execute(" create table %s using stb tags(%d) "%(tbname , tbnum))
ts = self.ts
for row in range(rownums):
ts = self.ts + time_step*row
c1 = random.randint(0,1000)
c2 = random.randint(0,100000)
c3 = random.randint(0,125)
c4 = random.randint(0,125)
c5 = random.random()/1.0
c6 = random.random()/1.0
c7 = "'true'"
c8 = "'binary_val'"
c9 = "'nchar_val'"
c10 = ts
tdSql.execute(f" insert into {tbname} values ({ts},{c1},{c2},{c3},{c4},{c5},{c6},{c7},{c8},{c9},{c10})")
tdSql.execute("use test")
tbnames = ["stb", "sub_tb_1"]
support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"]
for tbname in tbnames:
tdSql.query("desc {}".format(tbname))
coltypes = tdSql.queryResult
for coltype in coltypes:
colname = coltype[0]
if coltype[1] in support_types and coltype[-1] != "TAG" :
irate_sql = "select irate({}) from (select * from {} order by tbname ) ".format(colname, tbname)
origin_sql = "select ts , {} , cast(ts as bigint) from (select ts , {} from {} order by ts desc limit 2 offset 0 ) order by ts".format(colname,colname, tbname)
tdSql.query(irate_sql)
irate_result = tdSql.queryResult
tdSql.query(origin_sql)
origin_result = tdSql.queryResult
irate_value = irate_result[0][0]
if origin_result[1][-1] - origin_result[0][-1] == 0:
comput_irate_value = 0
elif (origin_result[1][1] - origin_result[0][1])<0:
comput_irate_value = origin_result[1][1]*1000/( origin_result[1][-1] - origin_result[0][-1])
else:
comput_irate_value = (origin_result[1][1] - origin_result[0][1])*1000/( origin_result[1][-1] - origin_result[0][-1])
if comput_irate_value ==irate_value:
tdLog.info(" irate work as expected , sql is %s "% irate_sql)
else:
tdLog.exit(" irate work not as expected , sql is %s "% irate_sql)
def prepare_tag_datas(self):
# prepare datas
tdSql.execute(
"create database if not exists testdb keep 3650 duration 1000")
tdSql.execute(" use testdb ")
tdSql.execute(
'''create table stb1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32))
'''
)
tdSql.execute(
'''
create table t1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
'''
)
for i in range(4):
tdSql.execute(
f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )')
for i in range(9):
tdSql.execute(
f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
"insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
tdSql.execute(
"insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute(
"insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
"insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
"insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
f'''insert into t1 values
( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
'''
)
def test_errors(self):
tdSql.execute("use testdb")
error_sql_lists = [
"select irate from t1",
"select irate(-+--+c1) from t1",
# "select +-irate(c1) from t1",
# "select ++-irate(c1) from t1",
# "select ++--irate(c1) from t1",
# "select - -irate(c1)*0 from t1",
# "select irate(tbname+1) from t1 ",
"select irate(123--123)==1 from t1",
"select irate(c1) as 'd1' from t1",
"select irate(c1 ,c2 ) from t1",
"select irate(c1 ,NULL) from t1",
"select irate(,) from t1;",
"select irate(irate(c1) ab from t1)",
"select irate(c1) as int from t1",
"select irate from stb1",
# "select irate(-+--+c1) from stb1",
# "select +-irate(c1) from stb1",
# "select ++-irate(c1) from stb1",
# "select ++--irate(c1) from stb1",
# "select - -irate(c1)*0 from stb1",
# "select irate(tbname+1) from stb1 ",
"select irate(123--123)==1 from stb1",
"select irate(c1) as 'd1' from stb1",
"select irate(c1 ,c2 ) from stb1",
"select irate(c1 ,NULL) from stb1",
"select irate(,) from stb1;",
"select irate(abs(c1) ab from stb1)",
"select irate(c1) as int from stb1"
]
for error_sql in error_sql_lists:
tdSql.error(error_sql)
def support_types(self):
tdSql.execute("use testdb")
tbnames = ["stb1", "t1", "ct1", "ct2"]
support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"]
for tbname in tbnames:
tdSql.query("desc {}".format(tbname))
coltypes = tdSql.queryResult
for coltype in coltypes:
colname = coltype[0]
irate_sql = "select irate({}) from {}".format(colname, tbname)
if coltype[1] in support_types:
tdSql.query(irate_sql)
else:
tdSql.error(irate_sql)
def basic_irate_function(self):
# used for empty table , ct3 is empty
tdSql.query("select irate(c1) from ct3")
tdSql.checkRows(0)
tdSql.query("select irate(c2) from ct3")
tdSql.checkRows(0)
# used for regular table
tdSql.query("select irate(c1) from t1")
tdSql.checkData(0, 0, 0.000000386)
# used for sub table
tdSql.query("select irate(abs(c1+c2)) from ct1")
tdSql.checkData(0, 0, 0.000000000)
# mix with common col
tdSql.error("select c1, irate(c1) from ct1")
# mix with common functions
tdSql.error("select irate(c1), abs(c1) from ct4 ")
# agg functions mix with agg functions
tdSql.query("select irate(c1), count(c5) from stb1 partition by tbname ")
tdSql.checkData(0, 0, 0.000000000)
tdSql.checkData(1, 0, 0.000000000)
tdSql.checkData(0, 1, 13)
tdSql.checkData(1, 1, 9)
def irate_func_filter(self):
tdSql.execute("use testdb")
tdSql.query(
"select irate(c1+2)/2 from ct4 where c1>5 ")
tdSql.checkRows(1)
tdSql.checkData(0, 0, 0.000000514)
tdSql.query(
"select irate(c1+c2)/10 from ct4 where c1=5 ")
tdSql.checkRows(1)
tdSql.checkData(0, 0, 0.000000000)
tdSql.query(
"select irate(c1+c2)/10 from stb1 where c1 = 5 partition by tbname ")
tdSql.checkRows(2)
tdSql.checkData(0, 0, 0.000000000)
def irate_Arithmetic(self):
pass
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
tdLog.printNoPrefix("==========step1:create table ==============")
self.prepare_tag_datas()
tdLog.printNoPrefix("==========step2:test errors ==============")
self.test_errors()
tdLog.printNoPrefix("==========step3:support types ============")
self.support_types()
tdLog.printNoPrefix("==========step4: irate basic query ============")
self.basic_irate_function()
tdLog.printNoPrefix("==========step5: irate filter query ============")
self.irate_func_filter()
tdLog.printNoPrefix("==========step6: check result of query ============")
self.insert_datas_and_check_irate(self.tb_nums,self.row_nums,self.time_step)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())
tests/system-test/2-query/json_tag.py
浏览文件 @ 086d6f39
......@@ -412,52 +412,59 @@ class TDTestCase:
tdSql.checkColNameList(res, cname_list)
#
# test group by & order by json tag
tdSql.query("select ts,jtag->'tag1' from jsons1 partition by jtag->'tag1' order by jtag->'tag1' desc")
tdSql.checkRows(11)
tdSql.checkData(0, 1, '"femail"')
tdSql.checkData(2, 1, '"收到货"')
tdSql.checkData(7, 1, "false")
# tdSql.error("select count(*) from jsons1 group by jtag")
# tdSql.error("select count(*) from jsons1 partition by jtag")
# tdSql.error("select count(*) from jsons1 group by jtag order by jtag")
tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag2'")
tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag")
tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc")
tdSql.checkRows(8)
tdSql.checkData(0, 0, 2)
tdSql.checkData(0, 1, '"femail"')
tdSql.checkData(1, 0, 2)
tdSql.checkData(1, 1, '"收到货"')
tdSql.checkData(2, 0, 1)
tdSql.checkData(2, 1, "11.000000000")
tdSql.checkData(5, 0, 1)
tdSql.checkData(5, 1, "false")
tdSql.checkData(6, 0, 1)
tdSql.checkData(6, 1, "null")
tdSql.checkData(7, 0, 2)
tdSql.checkData(7, 1, None)
tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' asc")
tdSql.checkRows(8)
tdSql.checkData(0, 0, 2)
tdSql.checkData(0, 1, None)
tdSql.checkData(2, 0, 1)
tdSql.checkData(2, 1, "false")
tdSql.checkData(5, 0, 1)
tdSql.checkData(5, 1, "11.000000000")
tdSql.checkData(7, 0, 2)
tdSql.checkData(7, 1, '"femail"')
# tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc")
# tdSql.checkRows(8)
# tdSql.checkData(0, 0, 2)
# tdSql.checkData(0, 1, '"femail"')
# tdSql.checkData(1, 0, 2)
# tdSql.checkData(1, 1, '"收到货"')
# tdSql.checkData(2, 0, 1)
# tdSql.checkData(2, 1, "11.000000000")
# tdSql.checkData(5, 0, 1)
# tdSql.checkData(5, 1, "false")
# tdSql.checkData(6, 0, 1)
# tdSql.checkData(6, 1, "null")
# tdSql.checkData(7, 0, 2)
# tdSql.checkData(7, 1, None)
# tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' asc")
# tdSql.checkRows(8)
# tdSql.checkData(0, 0, 2)
# tdSql.checkData(0, 1, None)
# tdSql.checkData(2, 0, 1)
# tdSql.checkData(2, 1, "false")
# tdSql.checkData(5, 0, 1)
# tdSql.checkData(5, 1, "11.000000000")
# tdSql.checkData(7, 0, 2)
# tdSql.checkData(7, 1, '"femail"')
#
# test stddev with group by json tag
tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'")
tdSql.checkRows(8)
tdSql.checkData(0, 0, 10)
tdSql.checkData(0, 1, None)
tdSql.checkData(4, 0, 0)
tdSql.checkData(4, 1, "5.000000000")
tdSql.checkData(7, 0, 11)
tdSql.checkData(7, 1, '"femail"')
res = tdSql.getColNameList("select stddev(dataint),jsons1.jtag->'tag1' from jsons1 group by jsons1.jtag->'tag1' order by jtag->'tag1'")
cname_list = []
cname_list.append("stddev(dataint)")
cname_list.append("jsons1.jtag->'tag1'")
tdSql.checkColNameList(res, cname_list)
# tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'")
# tdSql.checkRows(8)
# tdSql.checkData(0, 0, 10)
# tdSql.checkData(0, 1, None)
# tdSql.checkData(4, 0, 0)
# tdSql.checkData(4, 1, "5.000000000")
# tdSql.checkData(7, 0, 11)
# tdSql.checkData(7, 1, '"femail"')
#
# res = tdSql.getColNameList("select stddev(dataint),jsons1.jtag->'tag1' from jsons1 group by jsons1.jtag->'tag1' order by jtag->'tag1'")
# cname_list = []
# cname_list.append("stddev(dataint)")
# cname_list.append("jsons1.jtag->'tag1'")
# tdSql.checkColNameList(res, cname_list)
# test top/bottom with group by json tag
# tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'")
......@@ -470,8 +477,8 @@ class TDTestCase:
# tdSql.checkData(10, 1, '"femail"')
# test having
tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' having count(*) > 1")
tdSql.checkRows(3)
# tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' having count(*) > 1")
# tdSql.checkRows(3)
# subquery with json tag
tdSql.query("select * from (select jtag, dataint from jsons1) order by dataint")
......
tests/system-test/6-cluster/5dnode3mnodeStop.py
浏览文件 @ 086d6f39
......@@ -227,6 +227,7 @@ class TDTestCase:
# fisr add three mnodes;
tdSql.execute("create mnode on dnode 2")
time.sleep(10)
tdSql.execute("create mnode on dnode 3")
# fisrt check statut ready
......
tests/system-test/7-tmq/tmqCommon.py
浏览文件 @ 086d6f39
......@@ -11,7 +11,9 @@
# -*- coding: utf-8 -*-
from asyncore import loop
from collections import defaultdict
import subprocess
import random
import string
import threading
......@@ -75,7 +77,7 @@ class TMQCom:
return resultList
def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0):
def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0,alias=0):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
if valgrind == 1:
......@@ -88,30 +90,53 @@ class TMQCom:
shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName)
shellCmd += "> nul 2>&1 &"
else:
shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath
processorName = buildPath + '/build/bin/tmq_sim'
if alias != 0:
processorNameNew = buildPath + '/build/bin/tmq_sim_new'
shellCmd = 'cp %s %s'%(processorName, processorNameNew)
os.system(shellCmd)
processorName = processorNameNew
shellCmd = 'nohup ' + processorName + ' -c ' + cfgPath
shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName)
shellCmd += "> /dev/null 2>&1 &"
tdLog.info(shellCmd)
os.system(shellCmd)
def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'):
while 1:
os.system(shellCmd)
def stopTmqSimProcess(self, processorName):
psCmd = "ps -ef|grep -w %s|grep -v grep | awk '{print $2}'"%(processorName)
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -INT %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(0.2)
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
tdLog.debug("%s is stopped by kill -INT" % (processorName))
def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb',rows=1):
loopFlag = 1
while loopFlag:
tdSql.query("select * from %s.notifyinfo"%cdbName)
#tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0):
break
else:
time.sleep(0.1)
actRows = tdSql.getRows()
if (actRows >= rows):
for i in range(actRows):
if tdSql.getData(i, 1) == 0:
loopFlag = 0
break
time.sleep(0.1)
return
def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'):
while 1:
def getStartCommitNotifyFromTmqsim(self,cdbName='cdb',rows=2):
loopFlag = 1
while loopFlag:
tdSql.query("select * from %s.notifyinfo"%cdbName)
#tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
if tdSql.getRows() == 2 :
print(tdSql.getData(0, 1), tdSql.getData(1, 1))
if tdSql.getData(1, 1) == 1:
break
actRows = tdSql.getRows()
if (actRows >= rows):
for i in range(actRows):
if tdSql.getData(i, 1) == 1:
loopFlag = 0
break
time.sleep(0.1)
return
......
tests/system-test/7-tmq/tmqConsumerGroup.py 0 → 100644
浏览文件 @ 086d6f39
import taos
import sys
import time
import socket
import os
import threading
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
sys.path.append("./7-tmq")
from tmqCommon import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
#tdSql.init(conn.cursor(), logSql) # output sql.txt file
def checkFileContent(self, consumerId, queryString):
buildPath = tdCom.getBuildPath()
cfgPath = tdCom.getClientCfgPath()
dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
tdLog.info(cmdStr)
os.system(cmdStr)
consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
consumeFile = open(consumeRowsFile, mode='r')
queryFile = open(dstFile, mode='r')
# skip first line for it is schema
queryFile.readline()
while True:
dst = queryFile.readline()
src = consumeFile.readline()
if dst:
if dst != src:
tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
else:
break
return
def tmqCase1(self):
tdLog.printNoPrefix("======== test case 1: ")
paraDict = {'dbName': 'db1',
'dropFlag': 1,
'event': '',
'vgroups': 4,
'stbName': 'stb',
'colPrefix': 'c',
'tagPrefix': 't',
'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
'ctbPrefix': 'ctb',
'ctbNum': 10,
'rowsPerTbl': 1000,
'batchNum': 10,
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
'pollDelay': 20,
'showMsg': 1,
'showRow': 1}
topicNameList = ['topic1', 'topic2']
expectRowsList = []
tmqCom.initConsumerTable()
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
tdLog.info("create stb")
tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
tdLog.info("create ctb")
tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
tdLog.info("insert data")
tmqCom.insert_data_2(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
tdLog.info("create topics from stb with filter")
# queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName'])
queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s" %(paraDict['dbName'], paraDict['stbName'])
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString)
tdSql.query(queryString)
expectRowsList.append(tdSql.getRows())
# create one stb2
paraDict["stbName"] = 'stb2'
paraDict["ctbPrefix"] = 'ctbx'
paraDict["rowsPerTbl"] = 5000
tdLog.info("create stb2")
tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
tdLog.info("create ctb2")
tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
# queryString = "select ts, sin(c1), abs(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName'])
queryString = "select ts, sin(c1), abs(pow(c1,3)) from %s.%s" %(paraDict['dbName'], paraDict['stbName'])
sqlString = "create topic %s as %s" %(topicNameList[1], queryString)
tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString)
# tdSql.query(queryString)
# expectRowsList.append(tdSql.getRows())
# init consume info, and start tmq_sim, then check consume result
tdLog.info("insert consume info to consume processor")
consumerId = 0
expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
topicList = "%s,%s"%(topicNameList[0],topicNameList[1])
ifcheckdata = 1
ifManualCommit = 1
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:3000, auto.offset.reset:earliest'
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor 1")
tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
tdLog.info("start consume processor 2")
tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'],'cdb',0,1)
tdLog.info("async insert data")
pThread = tmqCom.asyncInsertData(paraDict)
tdLog.info("wait consumer commit notify")
tmqCom.getStartCommitNotifyFromTmqsim(rows=4)
tdLog.info("pkill one consume processor")
tmqCom.stopTmqSimProcess('tmq_sim_new')
pThread.join()
tdLog.info("wait the consume result")
expectRows = 2
resultList = tmqCom.selectConsumeResult(expectRows)
actTotalRows = 0
for i in range(len(resultList)):
actTotalRows += resultList[i]
tdSql.query(queryString)
expectRowsList.append(tdSql.getRows())
expectTotalRows = 0
for i in range(len(expectRowsList)):
expectTotalRows += expectRowsList[i]
tdLog.info("act consume rows: %d, expect consume rows: %d"%(actTotalRows, expectTotalRows))
if expectTotalRows <= resultList[0]:
tdLog.info("act consume rows: %d should >= expect consume rows: %d"%(actTotalRows, expectTotalRows))
tdLog.exit("0 tmq consume rows error!")
# time.sleep(10)
# for i in range(len(topicNameList)):
# tdSql.query("drop topic %s"%topicNameList[i])
tdLog.printNoPrefix("======== test case 1 end ...... ")
def run(self):
tdSql.prepare()
self.tmqCase1()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
event = threading.Event()
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())
tests/system-test/fulltest.sh
浏览文件 @ 086d6f39
......@@ -109,6 +109,9 @@ python3 ./test.py -f 2-query/distribute_agg_apercentile.py
python3 ./test.py -f 2-query/distribute_agg_avg.py
python3 ./test.py -f 2-query/distribute_agg_stddev.py
python3 ./test.py -f 2-query/twa.py
python3 ./test.py -f 2-query/irate.py
python3 ./test.py -f 2-query/function_null.py
python3 ./test.py -f 6-cluster/5dnode1mnode.py
python3 ./test.py -f 6-cluster/5dnode2mnode.py
......@@ -135,3 +138,4 @@ python3 ./test.py -f 7-tmq/stbFilter.py
python3 ./test.py -f 7-tmq/tmqCheckData.py
python3 ./test.py -f 7-tmq/tmqUdf.py
#python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5
python3 ./test.py -f 7-tmq/tmqConsumerGroup.py
tests/test/c/tmqSim.c
浏览文件 @ 086d6f39
......@@ -93,8 +93,6 @@ static SConfInfo g_stConfInfo;
TdFilePtr g_fp = NULL;
static int running = 1;
int8_t useSnapshot = 0;
// char* g_pRowValue = NULL;
// TdFilePtr g_fp = NULL;
......@@ -126,11 +124,23 @@ char* getCurrentTimeString(char* timeString) {
return timeString;
}
static void tmqStop(int signum, void *info, void *ctx) {
running = 0;
char tmpString[128];
taosFprintfFile(g_fp, "%s tmqStop() receive stop signal[%d]\n", getCurrentTimeString(tmpString), signum);
}
static void tmqSetSignalHandle() {
taosSetSignal(SIGINT, tmqStop);
}
void initLogFile() {
char filename[256];
char tmpString[128];
sprintf(filename, "%s/../log/tmqlog_%s.txt", configDir, getCurrentTimeString(tmpString));
pid_t process_id = getpid();
sprintf(filename, "%s/../log/tmqlog-%d-%s.txt", configDir, process_id, getCurrentTimeString(tmpString));
#ifdef WINDOWS
for (int i = 2; i < sizeof(filename); i++) {
if (filename[i] == ':') filename[i] = '-';
......@@ -205,7 +215,7 @@ void parseArgument(int32_t argc, char* argv[]) {
} else if (strcmp(argv[i], "-y") == 0) {
g_stConfInfo.consumeDelay = atol(argv[++i]);
} else if (strcmp(argv[i], "-e") == 0) {
useSnapshot = (int8_t)atol(argv[++i]);
g_stConfInfo.useSnapshot = atol(argv[++i]);
} else {
pError("%s unknow para: %s %s", GREEN, argv[++i], NC);
exit(-1);
......@@ -519,7 +529,9 @@ static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) {
g_once_commit_flag = 1;
notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT);
}
taosFprintfFile(g_fp, "tmq_commit_cb_print() be called\n");
char tmpString[128];
taosFprintfFile(g_fp, "%s tmq_commit_cb_print() be called\n", getCurrentTimeString(tmpString));
}
void build_consumer(SThreadInfo* pInfo) {
......@@ -552,7 +564,7 @@ void build_consumer(SThreadInfo* pInfo) {
// tmq_conf_set(conf, "auto.offset.reset", "earliest");
// tmq_conf_set(conf, "auto.offset.reset", "latest");
//
if (useSnapshot) {
if (g_stConfInfo.useSnapshot) {
tmq_conf_set(conf, "experiment.use.snapshot", "true");
}
......@@ -651,6 +663,10 @@ void loop_consume(SThreadInfo* pInfo) {
}
}
if (0 == running) {
taosFprintfFile(g_fp, "receive stop signal and not continue consume\n");
}
pInfo->consumeMsgCnt = totalMsgs;
pInfo->consumeRowCnt = totalRows;
......@@ -680,7 +696,7 @@ void* consumeThreadFunc(void* param) {
int32_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList);
if (err != 0) {
pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err));
taosFprintfFile(g_fp, "tmq_subscribe()! reason: %s\n", tmq_err2str(err));
taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err));
assert(0);
return NULL;
}
......@@ -829,6 +845,8 @@ int main(int32_t argc, char* argv[]) {
getConsumeInfo();
saveConfigToLogFile();
tmqSetSignalHandle();
TdThreadAttr thattr;
taosThreadAttrInit(&thattr);
taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或