diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh
index d114d5eef810c0590e0d2ae90cfd27e4952f5ddc..e4d2d71b014293e1e328112e0d5e6e5677b772c0 100755
--- a/packaging/tools/makepkg.sh
+++ b/packaging/tools/makepkg.sh
@@ -114,6 +114,25 @@ mkdir -p ${install_dir}/examples
examples_dir="${top_dir}/tests/examples"
cp -r ${examples_dir}/c ${install_dir}/examples
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
+ if [ -d ${examples_dir}/JDBC/connectionPools/target ]; then
+ rm -rf ${examples_dir}/JDBC/connectionPools/target
+ fi
+ if [ -d ${examples_dir}/JDBC/JDBCDemo/target ]; then
+ rm -rf ${examples_dir}/JDBC/JDBCDemo/target
+ fi
+ if [ -d ${examples_dir}/JDBC/mybatisplus-demo/target ]; then
+ rm -rf ${examples_dir}/JDBC/mybatisplus-demo/target
+ fi
+ if [ -d ${examples_dir}/JDBC/springbootdemo/target ]; then
+ rm -rf ${examples_dir}/JDBC/springbootdemo/target
+ fi
+ if [ -d ${examples_dir}/JDBC/SpringJdbcTemplate/target ]; then
+ rm -rf ${examples_dir}/JDBC/SpringJdbcTemplate/target
+ fi
+ if [ -d ${examples_dir}/JDBC/taosdemo/target ]; then
+ rm -rf ${examples_dir}/JDBC/taosdemo/target
+ fi
+
cp -r ${examples_dir}/JDBC ${install_dir}/examples
cp -r ${examples_dir}/matlab ${install_dir}/examples
cp -r ${examples_dir}/python ${install_dir}/examples
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
index d85c8e5871ba070afc74418b056420134a44f627..a63225ab324cb1eefc9f221423c1018e8e1528eb 100644
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -1,6 +1,5 @@
name: tdengine
base: core18
-
version: '2.1.1.0'
icon: snap/gui/t-dengine.svg
summary: an open-source big data platform designed and optimized for IoT.
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index acfc1b0cf54e4c6f4485c98d4e583bfc12475f4c..61b659f96ccb2ca54e326e3178467ac5cfd6c6fd 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -7187,6 +7187,11 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, int32_t index) {
const char* msg1 = "point interpolation query needs timestamp";
const char* msg2 = "too many tables in from clause";
const char* msg3 = "start(end) time of query range required or time range too large";
+ // const char* msg5 = "too many columns in selection clause";
+ // const char* msg6 = "too many tables in from clause";
+ // const char* msg7 = "invalid table alias name";
+ // const char* msg8 = "alias name too long";
+ const char* msg9 = "only tag query not compatible with normal column filter";
int32_t code = TSDB_CODE_SUCCESS;
@@ -7326,6 +7331,20 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, int32_t index) {
}
}
+ if (tscQueryTags(pQueryInfo)) {
+ SExprInfo* pExpr1 = tscSqlExprGet(pQueryInfo, 0);
+
+ if (pExpr1->base.functionId != TSDB_FUNC_TID_TAG) {
+ int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryInfo->colList);
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ SColumn* pCols = taosArrayGetP(pQueryInfo->colList, i);
+ if (pCols->info.flist.numOfFilters > 0) {
+ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9);
+ }
+ }
+ }
+ }
+
// parse the having clause in the first place
if (validateHavingClause(pQueryInfo, pSqlNode->pHaving, pCmd, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery) !=
TSDB_CODE_SUCCESS) {
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 9b7d5c0c7fa79ef6937239d8604d40d4cedd6278..a2afaf286fb3cc8e225d957a7a7f1beb5996a3c5 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -1928,8 +1928,9 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
}
}
- tscDebug("0x%"PRIx64" recv table meta, uid:%" PRIu64 ", tid:%d, name:%s", pSql->self, pTableMeta->id.uid, pTableMeta->id.tid,
- tNameGetTableName(&pTableMetaInfo->name));
+ tscDebug("0x%"PRIx64" recv table meta, uid:%" PRIu64 ", tid:%d, name:%s, numOfCols:%d, numOfTags:%d", pSql->self,
+ pTableMeta->id.uid, pTableMeta->id.tid, tNameGetTableName(&pTableMetaInfo->name), pTableMeta->tableInfo.numOfColumns,
+ pTableMeta->tableInfo.numOfTags);
free(pTableMeta);
return TSDB_CODE_SUCCESS;
@@ -2072,7 +2073,7 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups;
if (pInfo->vgroupList->numOfVgroups <= 0) {
- tscDebug("0x%"PRIx64" empty vgroup info, no corresponding tables for stable", pSql->self);
+ tscDebug("0x%" PRIx64 " empty vgroup info, no corresponding tables for stable", pSql->self);
} else {
for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) {
// just init, no need to lock
diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c
index 17bf575b60eed9f71e72a608d0aef4944c887bef..2207a1ed16b010105b1746fbb3ac72a198b6341a 100644
--- a/src/client/src/tscStream.c
+++ b/src/client/src/tscStream.c
@@ -627,6 +627,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p
if (pSql->sqlstr == NULL) {
tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self);
tscFreeSqlObj(pSql);
+ free(pStream);
return NULL;
}
diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index 6928058f2301b05f7eda0b4a2b77f0d0edf0f45f..f02bd2254037ce7a10348b776e923ba2d5548c47 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -215,7 +215,7 @@ static void tscProcessSubscriptionTimer(void *handle, void *tmrId) {
taosTmrReset(tscProcessSubscriptionTimer, pSub->interval, pSub, tscTmr, &pSub->pTimer);
}
-
+//TODO refactor: extract table list name not simply from the sql
static SArray* getTableList( SSqlObj* pSql ) {
const char* p = strstr( pSql->sqlstr, " from " );
assert(p != NULL); // we are sure this is a 'select' statement
@@ -224,11 +224,11 @@ static SArray* getTableList( SSqlObj* pSql ) {
SSqlObj* pNew = taos_query(pSql->pTscObj, sql);
if (pNew == NULL) {
- tscError("0x%"PRIx64"failed to retrieve table id: cannot create new sql object.", pSql->self);
+ tscError("0x%"PRIx64" failed to retrieve table id: cannot create new sql object.", pSql->self);
return NULL;
} else if (taos_errno(pNew) != TSDB_CODE_SUCCESS) {
- tscError("0x%"PRIx64"failed to retrieve table id,error: %s", pSql->self, tstrerror(taos_errno(pNew)));
+ tscError("0x%"PRIx64" failed to retrieve table id,error: %s", pSql->self, tstrerror(taos_errno(pNew)));
return NULL;
}
diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h
index d3ac0ce28004c2b189ba98ac6266ec5dd1a7c935..0d32e17a4cd446b980f3bc464d12123dba6b9f19 100644
--- a/src/inc/taoserror.h
+++ b/src/inc/taoserror.h
@@ -219,6 +219,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_VND_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0512) //"Database write operation denied")
#define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) //"Database is syncing")
#define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) //"Invalid tsdb state")
+#define TSDB_CODE_VND_IS_CLOSING TAOS_DEF_ERROR_CODE(0, 0x0515) //"Database is closing")
// tsdb
#define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) //"Invalid table ID")
diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c
index b33e436aef07eeccfc864739a4ef495dd3cfc2f8..888dc4a04e4fc31d1b775e2f078f123899a61fa5 100644
--- a/src/kit/taosdemo/taosdemo.c
+++ b/src/kit/taosdemo/taosdemo.c
@@ -470,6 +470,7 @@ typedef struct SThreadInfo_S {
// seq of query or subscribe
uint64_t querySeq; // sequence number of sql command
+ TAOS_SUB* tsub;
} threadInfo;
diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c
index 9e3b87671e35e53ad5faa6ec7313ba2af3f6c247..84e4f33ca7efe2cafca9ec64d8b7872cac827403 100644
--- a/src/mnode/src/mnodeSdb.c
+++ b/src/mnode/src/mnodeSdb.c
@@ -719,13 +719,13 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *
if (action == SDB_ACTION_INSERT) {
return sdbPerformInsertAction(pHead, pTable);
} else if (action == SDB_ACTION_DELETE) {
- if (qtype == TAOS_QTYPE_FWD) {
+ //if (qtype == TAOS_QTYPE_FWD) {
// Drop database/stable may take a long time and cause a timeout, so we confirm first then reput it into queue
- sdbWriteFwdToQueue(1, hparam, TAOS_QTYPE_QUERY, unused);
- return TSDB_CODE_SUCCESS;
- } else {
+ // sdbWriteFwdToQueue(1, hparam, TAOS_QTYPE_QUERY, unused);
+ // return TSDB_CODE_SUCCESS;
+ //} else {
return sdbPerformDeleteAction(pHead, pTable);
- }
+ //}
} else if (action == SDB_ACTION_UPDATE) {
return sdbPerformUpdateAction(pHead, pTable);
} else {
diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index 4d3125a2d1ebe872e992846792e2b8042fb223a9..4e879537e431dba12a066d84abd3f867c6c00ada 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -1189,8 +1189,8 @@ static int32_t mnodeFindSuperTableTagIndex(SSTableObj *pStable, const char *tagN
static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
- mLInfo("msg:%p, app:%p stable %s, add tag result:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
- tstrerror(code));
+ mLInfo("msg:%p, app:%p stable %s, add tag result:%s, numOfTags:%d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
+ tstrerror(code), pStable->numOfTags);
return code;
}
diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c
index 67532ad85a965a18e5bed3f4c2bc40763bedcefc..a64d2564841241c7f49b7da82a48b51d27783c1a 100644
--- a/src/mnode/src/mnodeVgroup.c
+++ b/src/mnode/src/mnodeVgroup.c
@@ -121,7 +121,7 @@ static int32_t mnodeVgroupActionDelete(SSdbRow *pRow) {
SVgObj *pVgroup = pRow->pObj;
if (pVgroup->pDb == NULL) {
- mError("vgId:%d, db:%s is not exist while insert into hash", pVgroup->vgId, pVgroup->dbName);
+ mError("vgId:%d, db:%s is not exist while delete from hash", pVgroup->vgId, pVgroup->dbName);
return TSDB_CODE_MND_VGROUP_NOT_EXIST;
}
diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c
index 3d8f54ad9b643143235804a68825c0803c8ce81b..493d5f117ac4d213da2703ed60825209f19f46b4 100644
--- a/src/sync/src/syncMain.c
+++ b/src/sync/src/syncMain.c
@@ -709,7 +709,7 @@ static void syncChooseMaster(SSyncNode *pNode) {
}
static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
- int32_t onlineNum = 0;
+ int32_t onlineNum = 0, arbOnlineNum = 0;
int32_t masterIndex = -1;
int32_t replica = pNode->replica;
@@ -723,13 +723,15 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
if (pArb && pArb->role != TAOS_SYNC_ROLE_OFFLINE) {
onlineNum++;
+ ++arbOnlineNum;
replica = pNode->replica + 1;
}
if (onlineNum <= replica * 0.5) {
if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) {
- if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && onlineNum >= 1) {
+ if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && ((replica > 2 && onlineNum - arbOnlineNum > 1) || pNode->replica < 3)) {
sInfo("vgId:%d, self keep work as master, online:%d replica:%d", pNode->vgId, onlineNum, replica);
+ masterIndex = pNode->selfIndex;
} else {
nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
@@ -1002,6 +1004,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) {
if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
// nodeVersion = pHead->version;
code = (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
+ syncConfirmForward(pNode->rid, pHead->version, code, false);
} else {
if (nodeSStatus != TAOS_SYNC_STATUS_INIT) {
code = syncSaveIntoBuffer(pPeer, pHead);
@@ -1404,7 +1407,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) {
pthread_mutex_lock(&pNode->mutex);
for (int32_t i = 0; i < pSyncFwds->fwds; ++i) {
SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % SYNC_MAX_FWDS;
- if (ABS(time - pFwdInfo->time) < 2000) break;
+ if (ABS(time - pFwdInfo->time) < 10000) break;
sDebug("vgId:%d, forward info expired, hver:%" PRIu64 " curtime:%" PRIu64 " savetime:%" PRIu64, pNode->vgId,
pFwdInfo->version, time, pFwdInfo->time);
diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c
index 57d45cc8c00dab5b760f8e915b967e37217f9b64..569f9b01bda6d3a8cf5de6ba4b089e7bab49ca9f 100644
--- a/src/util/src/tcache.c
+++ b/src/util/src/tcache.c
@@ -613,7 +613,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) {
// todo memory leak if there are object with refcount greater than 0 in hash table?
taosHashCleanup(pCacheObj->pHashTable);
- taosTrashcanEmpty(pCacheObj, true);
+ taosTrashcanEmpty(pCacheObj, false);
__cache_lock_destroy(pCacheObj);
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index f133101ccad3c7eaa7bd206f3355ff5ff8b82298..fdf215a966582254978daf5997234efa9d2cfb8b 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -454,7 +454,11 @@ void vnodeDestroy(SVnodeObj *pVnode) {
}
if (pVnode->tsdb) {
- code = tsdbCloseRepo(pVnode->tsdb, 1);
+ // the deleted vnode does not need to commit, so as to speed up the deletion
+ int toCommit = 1;
+ if (pVnode->dropped) toCommit = 0;
+
+ code = tsdbCloseRepo(pVnode->tsdb, toCommit);
pVnode->tsdb = NULL;
}
diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c
index e5a1964915646c7d6da106b003abbc9909ed71db..4197428fec6b5d24e7791b2a5f8cb7df229cbca5 100644
--- a/src/vnode/src/vnodeSync.c
+++ b/src/vnode/src/vnodeSync.c
@@ -126,11 +126,16 @@ void vnodeStopSyncFile(int32_t vgId, uint64_t fversion) {
}
void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code) {
- void *pVnode = vnodeAcquire(vgId);
+ SVnodeObj *pVnode = vnodeAcquire(vgId);
if (pVnode == NULL) {
vError("vgId:%d, vnode not found while confirm forward", vgId);
}
+ if (code == TSDB_CODE_SYN_CONFIRM_EXPIRED && pVnode->status == TAOS_VN_STATUS_CLOSING) {
+ vDebug("vgId:%d, db:%s, vnode is closing while confirm forward", vgId, pVnode->db);
+ code = TSDB_CODE_VND_IS_CLOSING;
+ }
+
dnodeSendRpcVWriteRsp(pVnode, wparam, code);
vnodeRelease(pVnode);
}
diff --git a/tests/mas/Jenkinsfile b/tests/mas/Jenkinsfile
new file mode 100644
index 0000000000000000000000000000000000000000..b2a1a5e1167844e777909cc9688186d85b90a707
--- /dev/null
+++ b/tests/mas/Jenkinsfile
@@ -0,0 +1,309 @@
+def pre_test(){
+
+ sh '''
+ sudo rmtaos||echo 'no taosd installed'
+ '''
+ sh '''
+ cd ${WKC}
+ git reset --hard
+ git checkout $BRANCH_NAME
+ git pull
+ git submodule update
+ cd ${WK}
+ git reset --hard
+ git checkout $BRANCH_NAME
+ git pull
+ export TZ=Asia/Harbin
+ date
+ rm -rf ${WK}/debug
+ mkdir debug
+ cd debug
+ cmake .. > /dev/null
+ make > /dev/null
+ make install > /dev/null
+ pip3 install ${WKC}/src/connector/python/linux/python3/
+ '''
+ return 1
+}
+pipeline {
+ agent none
+ environment{
+
+ WK = '/var/lib/jenkins/workspace/TDinternal'
+ WKC= '/var/lib/jenkins/workspace/TDinternal/community'
+ }
+
+ stages {
+ stage('Parallel test stage') {
+ parallel {
+ stage('pytest') {
+ agent{label 'slam1'}
+ steps {
+ pre_test()
+ sh '''
+ cd ${WKC}/tests
+ find pytest -name '*'sql|xargs rm -rf
+ ./test-all.sh pytest
+ date'''
+ }
+ }
+ stage('test_b1') {
+ agent{label 'slam2'}
+ steps {
+ pre_test()
+
+ sh '''
+ cd ${WKC}/tests
+ ./test-all.sh b1
+ date'''
+
+
+ }
+ }
+
+ stage('test_crash_gen') {
+ agent{label "slam3"}
+ steps {
+ pre_test()
+ sh '''
+ cd ${WKC}/tests/pytest
+ '''
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WKC}/tests/pytest
+ ./crash_gen.sh -a -p -t 4 -s 2000
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WKC}/tests/pytest
+ rm -rf /var/lib/taos/*
+ rm -rf /var/log/taos/*
+ ./handle_crash_gen_val_log.sh
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WKC}/tests/pytest
+ rm -rf /var/lib/taos/*
+ rm -rf /var/log/taos/*
+ ./handle_taosd_val_log.sh
+ '''
+ }
+
+ sh'''
+ systemctl start taosd
+ sleep 10
+ '''
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WKC}/tests/gotest
+ bash batchtest.sh
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WKC}/tests/examples/python/PYTHONConnectorChecker
+ python3 PythonChecker.py
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WKC}/tests/examples/JDBC/JDBCDemo/
+ mvn clean package assembly:single -DskipTests >/dev/null
+ java -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WKC}/src/connector/jdbc
+ mvn clean package -Dmaven.test.skip=true >/dev/null
+ cd ${WKC}/tests/examples/JDBC/JDBCDemo/
+ java --class-path=../../../../src/connector/jdbc/target:$JAVA_HOME/jre/lib/ext -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cp -rf ${WKC}/tests/examples/nodejs ${JENKINS_HOME}/workspace/
+ cd ${JENKINS_HOME}/workspace/nodejs
+ node nodejsChecker.js host=localhost
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC#
+ dotnet run
+ '''
+ }
+ sh '''
+ systemctl stop taosd
+ cd ${WKC}/tests
+ ./test-all.sh b2
+ date
+ '''
+ sh '''
+ cd ${WKC}/tests
+ ./test-all.sh full unit
+ date'''
+ }
+ }
+
+ stage('test_valgrind') {
+ agent{label "slam4"}
+
+ steps {
+ pre_test()
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WKC}/tests/pytest
+ nohup taosd >/dev/null &
+ sleep 10
+ python3 concurrent_inquiry.py -c 1
+
+ '''
+ }
+ sh '''
+ cd ${WKC}/tests
+ ./test-all.sh full jdbc
+ date'''
+ sh '''
+ cd ${WKC}/tests/pytest
+ ./valgrind-test.sh 2>&1 > mem-error-out.log
+ ./handle_val_log.sh
+
+ date
+ cd ${WKC}/tests
+ ./test-all.sh b3
+ date'''
+ sh '''
+ date
+ cd ${WKC}/tests
+ ./test-all.sh full example
+ date'''
+ }
+ }
+
+ stage('arm64_build'){
+ agent{label 'arm64'}
+ steps{
+ sh '''
+ cd ${WK}
+ git fetch
+ git checkout develop
+ git pull
+ cd ${WKC}
+ git fetch
+ git checkout develop
+ git pull
+ git submodule update
+ cd ${WKC}/packaging
+ ./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0
+
+ '''
+ }
+ }
+ stage('arm32_build'){
+ agent{label 'arm32'}
+ steps{
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WK}
+ git fetch
+ git checkout develop
+ git pull
+ cd ${WKC}
+ git fetch
+ git checkout develop
+ git pull
+ git submodule update
+ cd ${WKC}/packaging
+ ./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0
+
+ '''
+ }
+
+ }
+ }
+ }
+ }
+
+ }
+ post {
+ success {
+ emailext (
+ subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
+ body: '''
+
+
+
+
+
+
+
+
+ 构建信息
+
|
+
+
+
+
+
+ - 构建名称>>分支:${PROJECT_NAME}
+ - 构建结果: Successful
+ - 构建编号:${BUILD_NUMBER}
+ - 触发用户:${CAUSE}
+ - 变更概要:${CHANGES}
+ - 构建地址:${BUILD_URL}
+ - 构建日志:${BUILD_URL}console
+ - 变更集:${JELLY_SCRIPT}
+
+
+ |
+
+
+
+ ''',
+ to: "yqliu@taosdata.com,pxiao@taosdata.com",
+ from: "support@taosdata.com"
+ )
+ }
+ failure {
+ emailext (
+ subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
+ body: '''
+
+
+
+
+
+
+
+
+ 构建信息
+
|
+
+
+
+
+
+ - 构建名称>>分支:${PROJECT_NAME}
+ - 构建结果: Successful
+ - 构建编号:${BUILD_NUMBER}
+ - 触发用户:${CAUSE}
+ - 变更概要:${CHANGES}
+ - 构建地址:${BUILD_URL}
+ - 构建日志:${BUILD_URL}console
+ - 变更集:${JELLY_SCRIPT}
+
+
+ |
+
+
+
+ ''',
+ to: "yqliu@taosdata.com,pxiao@taosdata.com",
+ from: "support@taosdata.com"
+ )
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh
index c6d4687ed700fc266e4911019ccf7882c086bc0f..bf62f401bffa1cd4d3189d551d02c8a3e227be4b 100755
--- a/tests/perftest-scripts/perftest-query.sh
+++ b/tests/perftest-scripts/perftest-query.sh
@@ -64,18 +64,25 @@ function runQueryPerfTest {
[ -f $PERFORMANCE_TEST_REPORT ] && rm $PERFORMANCE_TEST_REPORT
nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/taosperf/ > /dev/null 2>&1 &
echoInfo "Wait TDengine to start"
- sleep 300
+ sleep 60
echoInfo "Run Performance Test"
cd $WORK_DIR/TDengine/tests/pytest
python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
+ mkdir -p /var/lib/perf/
+ mkdir -p /var/log/perf/
+ rm -rf /var/lib/perf/*
+ rm -rf /var/log/perf/*
+ nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/perf/ > /dev/null 2>&1 &
+ echoInfo "Wait TDengine to start"
+ sleep 10
+ echoInfo "Run Performance Test"
+ cd $WORK_DIR/TDengine/tests/pytest
+
python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
-
- #python3 perfbenchmark/joinPerformance.py | tee -a $PERFORMANCE_TEST_REPORT
-
}
diff --git a/tests/pytest/crash_gen/service_manager.py b/tests/pytest/crash_gen/service_manager.py
index 95507f0142036f36f9a5249a48800bd481513844..1cd65c1dde4e9c2027a82730cf4fc12290048bbe 100644
--- a/tests/pytest/crash_gen/service_manager.py
+++ b/tests/pytest/crash_gen/service_manager.py
@@ -22,7 +22,7 @@ from queue import Queue, Empty
from .shared.config import Config
from .shared.db import DbTarget, DbConn
from .shared.misc import Logging, Helper, CrashGenError, Status, Progress, Dice
-from .shared.types import DirPath
+from .shared.types import DirPath, IpcStream
# from crash_gen.misc import CrashGenError, Dice, Helper, Logging, Progress, Status
# from crash_gen.db import DbConn, DbTarget
@@ -177,13 +177,12 @@ quorum 2
return "127.0.0.1"
def getServiceCmdLine(self): # to start the instance
- cmdLine = []
if Config.getConfig().track_memory_leaks:
Logging.info("Invoking VALGRIND on service...")
- cmdLine = ['valgrind', '--leak-check=yes']
- # TODO: move "exec -c" into Popen(), we can both "use shell" and NOT fork so ask to lose kill control
- cmdLine += ["exec " + self.getExecFile(), '-c', self.getCfgDir()] # used in subproce.Popen()
- return cmdLine
+ return ['exec /usr/bin/valgrind', '--leak-check=yes', self.getExecFile(), '-c', self.getCfgDir()]
+ else:
+ # TODO: move "exec -c" into Popen(), we can both "use shell" and NOT fork so ask to lose kill control
+ return ["exec " + self.getExecFile(), '-c', self.getCfgDir()] # used in subproce.Popen()
def _getDnodes(self, dbc):
dbc.query("show dnodes")
@@ -281,16 +280,16 @@ class TdeSubProcess:
return '[TdeSubProc: pid = {}, status = {}]'.format(
self.getPid(), self.getStatus() )
- def getStdOut(self) -> BinaryIO :
+ def getIpcStdOut(self) -> IpcStream :
if self._popen.universal_newlines : # alias of text_mode
raise CrashGenError("We need binary mode for STDOUT IPC")
# Logging.info("Type of stdout is: {}".format(type(self._popen.stdout)))
- return typing.cast(BinaryIO, self._popen.stdout)
+ return typing.cast(IpcStream, self._popen.stdout)
- def getStdErr(self) -> BinaryIO :
+ def getIpcStdErr(self) -> IpcStream :
if self._popen.universal_newlines : # alias of text_mode
raise CrashGenError("We need binary mode for STDERR IPC")
- return typing.cast(BinaryIO, self._popen.stderr)
+ return typing.cast(IpcStream, self._popen.stderr)
# Now it's always running, since we matched the life cycle
# def isRunning(self):
@@ -301,11 +300,6 @@ class TdeSubProcess:
def _start(self, cmdLine) -> Popen :
ON_POSIX = 'posix' in sys.builtin_module_names
-
- # Sanity check
- # if self.subProcess: # already there
- # raise RuntimeError("Corrupt process state")
-
# Prepare environment variables for coverage information
# Ref: https://stackoverflow.com/questions/2231227/python-subprocess-popen-with-a-modified-environment
@@ -314,9 +308,8 @@ class TdeSubProcess:
# print(myEnv)
# print("Starting TDengine with env: ", myEnv.items())
- # print("Starting TDengine via Shell: {}".format(cmdLineStr))
+ print("Starting TDengine: {}".format(cmdLine))
- # useShell = True # Needed to pass environments into it
return Popen(
' '.join(cmdLine), # ' '.join(cmdLine) if useShell else cmdLine,
shell=True, # Always use shell, since we need to pass ENV vars
@@ -732,19 +725,19 @@ class ServiceManagerThread:
self._ipcQueue = Queue() # type: Queue
self._thread = threading.Thread( # First thread captures server OUTPUT
target=self.svcOutputReader,
- args=(subProc.getStdOut(), self._ipcQueue, logDir))
+ args=(subProc.getIpcStdOut(), self._ipcQueue, logDir))
self._thread.daemon = True # thread dies with the program
self._thread.start()
time.sleep(0.01)
if not self._thread.is_alive(): # What happened?
- Logging.info("Failed to started process to monitor STDOUT")
+ Logging.info("Failed to start process to monitor STDOUT")
self.stop()
raise CrashGenError("Failed to start thread to monitor STDOUT")
Logging.info("Successfully started process to monitor STDOUT")
self._thread2 = threading.Thread( # 2nd thread captures server ERRORs
target=self.svcErrorReader,
- args=(subProc.getStdErr(), self._ipcQueue, logDir))
+ args=(subProc.getIpcStdErr(), self._ipcQueue, logDir))
self._thread2.daemon = True # thread dies with the program
self._thread2.start()
time.sleep(0.01)
@@ -887,14 +880,19 @@ class ServiceManagerThread:
print("\nNon-UTF8 server output: {}\n".format(bChunk.decode('cp437')))
return None
- def _textChunkGenerator(self, streamIn: BinaryIO, logDir: str, logFile: str
+ def _textChunkGenerator(self, streamIn: IpcStream, logDir: str, logFile: str
) -> Generator[TextChunk, None, None]:
'''
- Take an input stream with binary data, produced a generator of decoded
- "text chunks", and also save the original binary data in a log file.
+ Take an input stream with binary data (likely from Popen), produced a generator of decoded
+ "text chunks".
+
+ Side effect: it also save the original binary data in a log file.
'''
os.makedirs(logDir, exist_ok=True)
logF = open(os.path.join(logDir, logFile), 'wb')
+ if logF is None:
+ Logging.error("Failed to open log file (binary write): {}/{}".format(logDir, logFile))
+ return
for bChunk in iter(streamIn.readline, b''):
logF.write(bChunk) # Write to log file immediately
tChunk = self._decodeBinaryChunk(bChunk) # decode
@@ -902,14 +900,14 @@ class ServiceManagerThread:
yield tChunk # TODO: split into actual text lines
# At the end...
- streamIn.close() # Close the stream
- logF.close() # Close the output file
+ streamIn.close() # Close the incoming stream
+ logF.close() # Close the log file
- def svcOutputReader(self, stdOut: BinaryIO, queue, logDir: str):
+ def svcOutputReader(self, ipcStdOut: IpcStream, queue, logDir: str):
'''
The infinite routine that processes the STDOUT stream for the sub process being managed.
- :param stdOut: the IO stream object used to fetch the data from
+ :param ipcStdOut: the IO stream object used to fetch the data from
:param queue: the queue where we dump the roughly parsed chunk-by-chunk text data
:param logDir: where we should dump a verbatim output file
'''
@@ -917,7 +915,7 @@ class ServiceManagerThread:
# Important Reference: https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python
# print("This is the svcOutput Reader...")
# stdOut.readline() # Skip the first output? TODO: remove?
- for tChunk in self._textChunkGenerator(stdOut, logDir, 'stdout.log') :
+ for tChunk in self._textChunkGenerator(ipcStdOut, logDir, 'stdout.log') :
queue.put(tChunk) # tChunk garanteed not to be None
self._printProgress("_i")
@@ -940,12 +938,12 @@ class ServiceManagerThread:
Logging.info("EOF found TDengine STDOUT, marking the process as terminated")
self.setStatus(Status.STATUS_STOPPED)
- def svcErrorReader(self, stdErr: BinaryIO, queue, logDir: str):
+ def svcErrorReader(self, ipcStdErr: IpcStream, queue, logDir: str):
# os.makedirs(logDir, exist_ok=True)
# logFile = os.path.join(logDir,'stderr.log')
# fErr = open(logFile, 'wb')
# for line in iter(err.readline, b''):
- for tChunk in self._textChunkGenerator(stdErr, logDir, 'stderr.log') :
+ for tChunk in self._textChunkGenerator(ipcStdErr, logDir, 'stderr.log') :
queue.put(tChunk) # tChunk garanteed not to be None
# fErr.write(line)
Logging.info("TDengine STDERR: {}".format(tChunk))
diff --git a/tests/pytest/crash_gen/shared/types.py b/tests/pytest/crash_gen/shared/types.py
index 814a8219178d1c01ee0291447939984bdf81df9d..42fd2a1617cf729e4f23fc61a685027f738bc4a3 100644
--- a/tests/pytest/crash_gen/shared/types.py
+++ b/tests/pytest/crash_gen/shared/types.py
@@ -1,4 +1,4 @@
-from typing import Any, List, Dict, NewType
+from typing import Any, BinaryIO, List, Dict, NewType
from enum import Enum
DirPath = NewType('DirPath', str)
@@ -26,3 +26,5 @@ class TdDataType(Enum):
TdColumns = Dict[str, TdDataType]
TdTags = Dict[str, TdDataType]
+
+IpcStream = NewType('IpcStream', BinaryIO)
\ No newline at end of file
diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh
index a748c9dd2da623b2648b35ca7dd526dde8beebdb..d8e2a31e70bd14b8feef6c8a45df671473e68537 100755
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@@ -183,7 +183,7 @@ python3 ./test.py -f stable/query_after_reset.py
# perfbenchmark
python3 ./test.py -f perfbenchmark/bug3433.py
#python3 ./test.py -f perfbenchmark/bug3589.py
-
+python3 ./test.py -f perfbenchmark/taosdemoInsert.py
#query
python3 ./test.py -f query/filter.py
diff --git a/tests/pytest/insert/insertFromCSVPerformance.py b/tests/pytest/insert/insertFromCSVPerformance.py
index 84fd1d7ccab85d7e4469c779f10f047160ee0565..e1f741bd1203e93cd25832866f97be2e87d2b37d 100644
--- a/tests/pytest/insert/insertFromCSVPerformance.py
+++ b/tests/pytest/insert/insertFromCSVPerformance.py
@@ -31,7 +31,7 @@ class insertFromCSVPerformace:
self.host = "127.0.0.1"
self.user = "root"
self.password = "taosdata"
- self.config = "/etc/taosperf"
+ self.config = "/etc/perf"
self.conn = taos.connect(
self.host,
self.user,
diff --git a/tests/pytest/perfbenchmark/taosdemoInsert.py b/tests/pytest/perfbenchmark/taosdemoInsert.py
new file mode 100644
index 0000000000000000000000000000000000000000..59a8143d5a34014ecbdfe57c6ecc073c0eaeb39c
--- /dev/null
+++ b/tests/pytest/perfbenchmark/taosdemoInsert.py
@@ -0,0 +1,387 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import taos
+import sys
+import os
+import json
+import argparse
+import subprocess
+import datetime
+import re
+
+
+from multiprocessing import cpu_count
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.dnodes import TDDnode
+
+class Taosdemo:
+ def __init__(self, clearCache, dbName, keep):
+ self.clearCache = clearCache
+ self.dbname = dbName
+ self.drop = "yes"
+ self.keep = keep
+ self.host = "127.0.0.1"
+ self.user = "root"
+ self.password = "taosdata"
+ # self.config = "/etc/taosperf"
+ # self.conn = taos.connect(
+ # self.host,
+ # self.user,
+ # self.password,
+ # self.config)
+
+ # env config
+ def getBuildPath(self) -> str:
+ selfPath = os.path.dirname(os.path.realpath(__file__))
+
+ if ("community" in selfPath):
+ projPath = selfPath[:selfPath.find("community")]
+ else:
+ projPath = selfPath[:selfPath.find("tests")]
+
+ for root, dirs, files in os.walk(projPath):
+ if ("taosd" in files):
+ rootRealPath = os.path.dirname(os.path.realpath(root))
+ if ("packaging" not in rootRealPath):
+ buildPath = root[:len(root) - len("/debug/build/bin")]
+ break
+ return buildPath
+
+ def getExeToolsDir(self) -> str:
+ self.debugdir = self.getBuildPath() + "/debug/build/bin"
+ return self.debugdir
+
+ def getCfgDir(self) -> str:
+ self.config = self.getBuildPath() + "/sim/dnode1/cfg"
+ return self.config
+
+ # taodemo insert file config
+ def dbinfocfg(self) -> dict:
+ return {
+ "name": self.dbname,
+ "drop": self.drop,
+ "replica": 1,
+ "days": 10,
+ "cache": 16,
+ "blocks": 8,
+ "precision": "ms",
+ "keep": self.keep,
+ "minRows": 100,
+ "maxRows": 4096,
+ "comp": 2,
+ "walLevel": 1,
+ "cachelast": 0,
+ "quorum": 1,
+ "fsync": 3000,
+ "update": 0
+ }
+
+ def type_check(func):
+ def wrapper(self, **kwargs):
+ num_types = ["int", "float", "bigint", "tinyint", "smallint", "double"]
+ str_types = ["binary", "nchar"]
+ for k ,v in kwargs.items():
+ if k.lower() not in num_types and k.lower() not in str_types:
+ return f"args {k} type error, not allowed"
+ elif not isinstance(v, (int, list, tuple)):
+ return f"value {v} type error, not allowed"
+ elif k.lower() in num_types and not isinstance(v, int):
+ return f"arg {v} takes 1 positional argument must be type int "
+ elif isinstance(v, (list,tuple)) and len(v) > 2:
+ return f"arg {v} takes from 1 to 2 positional arguments but more than 2 were given "
+ elif isinstance(v,(list,tuple)) and [ False for _ in v if not isinstance(_, int) ]:
+ return f"arg {v} takes from 1 to 2 positional arguments must be type int "
+ else:
+ pass
+ return func(self, **kwargs)
+ return wrapper
+
+ @type_check
+ def column_tag_count(self, **column_tag) -> list :
+ init_column_tag = []
+ for k, v in column_tag.items():
+ if re.search(k, "int, float, bigint, tinyint, smallint, double", re.IGNORECASE):
+ init_column_tag.append({"type": k, "count": v})
+ elif re.search(k, "binary, nchar", re.IGNORECASE):
+ if isinstance(v, int):
+ init_column_tag.append({"type": k, "count": v, "len":8})
+ elif len(v) == 1:
+ init_column_tag.append({"type": k, "count": v[0], "len": 8})
+ else:
+ init_column_tag.append({"type": k, "count": v[0], "len": v[1]})
+ return init_column_tag
+
+ def stbcfg(self, stb: str, child_tab_count: int, rows: int, prechildtab: str, columns: dict, tags: dict) -> dict:
+ return {
+ "name": stb,
+ "child_table_exists": "no",
+ "childtable_count": child_tab_count,
+ "childtable_prefix": prechildtab,
+ "auto_create_table": "no",
+ "batch_create_tbl_num": 10,
+ "data_source": "rand",
+ "insert_mode": "taosc",
+ "insert_rows": rows,
+ "childtable_limit": 0,
+ "childtable_offset": 0,
+ "rows_per_tbl": 1,
+ "max_sql_len": 65480,
+ "disorder_ratio": 0,
+ "disorder_range": 1000,
+ "timestamp_step": 10,
+ "start_timestamp": f"{datetime.datetime.now():%F %X}",
+ "sample_format": "csv",
+ "sample_file": "./sample.csv",
+ "tags_file": "",
+ "columns": self.column_tag_count(**columns),
+ "tags": self.column_tag_count(**tags)
+ }
+
+ def schemecfg(self,intcount=1,floatcount=0,bcount=0,tcount=0,scount=0,doublecount=0,binarycount=0,ncharcount=0):
+ return {
+ "INT": intcount,
+ "FLOAT": floatcount,
+ "BIGINT": bcount,
+ "TINYINT": tcount,
+ "SMALLINT": scount,
+ "DOUBLE": doublecount,
+ "BINARY": binarycount,
+ "NCHAR": ncharcount
+ }
+
+ def insertcfg(self,db: dict, stbs: list) -> dict:
+ return {
+ "filetype": "insert",
+ "cfgdir": self.config,
+ "host": self.host,
+ "port": 6030,
+ "user": self.user,
+ "password": self.password,
+ "thread_count": cpu_count(),
+ "thread_count_create_tbl": cpu_count(),
+ "result_file": "/tmp/insert_res.txt",
+ "confirm_parameter_prompt": "no",
+ "insert_interval": 0,
+ "num_of_records_per_req": 100,
+ "max_sql_len": 1024000,
+ "databases": [{
+ "dbinfo": db,
+ "super_tables": stbs
+ }]
+ }
+
+ def createinsertfile(self,db: dict, stbs: list) -> str:
+ date = datetime.datetime.now()
+ file_create_table = f"/tmp/insert_{date:%F-%H%M}.json"
+
+ with open(file_create_table, 'w') as f:
+ json.dump(self.insertcfg(db, stbs), f)
+
+ return file_create_table
+
+ # taosdemo query file config
+ def querysqls(self, sql: str) -> list:
+ return [{"sql":sql,"result":""}]
+
+ def querycfg(self, sql: str) -> dict:
+ return {
+ "filetype": "query",
+ "cfgdir": self.config,
+ "host": self.host,
+ "port": 6030,
+ "user": self.user,
+ "password": self.password,
+ "confirm_parameter_prompt": "yes",
+ "query_times": 10,
+ "query_mode": "taosc",
+ "databases": self.dbname,
+ "specified_table_query": {
+ "query_interval": 0,
+ "concurrent": cpu_count(),
+ "sqls": self.querysqls(sql)
+ }
+ }
+
+ def createqueryfile(self, sql: str):
+ date = datetime.datetime.now()
+ file_query_table = f"/tmp/query_{date:%F-%H%M}.json"
+
+ with open(file_query_table,"w") as f:
+ json.dump(self.querycfg(sql), f)
+
+ return file_query_table
+
+ # Execute taosdemo, and delete temporary files when finished
+ def taosdemotable(self, filepath: str, resultfile="/dev/null"):
+ taosdemopath = self.getBuildPath() + "/debug/build/bin"
+ with open(filepath,"r") as f:
+ filetype = json.load(f)["filetype"]
+ if filetype == "insert":
+ taosdemo_table_cmd = f"{taosdemopath}/taosdemo -f {filepath} > {resultfile} 2>&1"
+ else:
+ taosdemo_table_cmd = f"yes | {taosdemopath}/taosdemo -f {filepath} > {resultfile} 2>&1"
+ try:
+ _ = subprocess.check_output(taosdemo_table_cmd, shell=True).decode("utf-8")
+ except subprocess.CalledProcessError as e:
+ _ = e.output
+
+ def droptmpfile(self, filepath: str):
+ drop_file_cmd = f"[ -f {filepath} ] && rm -f {filepath}"
+ try:
+ _ = subprocess.check_output(drop_file_cmd, shell=True).decode("utf-8")
+ except subprocess.CalledProcessError as e:
+ _ = e.output
+
+ # TODO:需要完成TD-4153的数据插入和客户端请求的性能查询。
+ def td4153insert(self):
+
+ tdLog.printNoPrefix("========== start to create table and insert data ==========")
+ self.dbname = "td4153"
+ db = self.dbinfocfg()
+ stblist = []
+
+ columntype = self.schemecfg(intcount=1, ncharcount=100)
+ tagtype = self.schemecfg(intcount=1)
+ stbname = "stb1"
+ prechild = "t1"
+ stable = self.stbcfg(
+ stb=stbname,
+ prechildtab=prechild,
+ child_tab_count=2,
+ rows=10000,
+ columns=columntype,
+ tags=tagtype
+ )
+ stblist.append(stable)
+ insertfile = self.createinsertfile(db=db, stbs=stblist)
+
+ nmon_file = f"/tmp/insert_{datetime.datetime.now():%F-%H%M}.nmon"
+ cmd = f"nmon -s5 -F {nmon_file} -m /tmp/"
+ try:
+ _ = subprocess.check_output(cmd, shell=True).decode("utf-8")
+ except subprocess.CalledProcessError as e:
+ _ = e.output
+
+ self.taosdemotable(insertfile)
+ self.droptmpfile(insertfile)
+ self.droptmpfile("/tmp/insert_res.txt")
+
+ # In order to prevent too many performance files from being generated, the nmon file is deleted.
+ # and the delete statement can be cancelled during the actual test.
+ self.droptmpfile(nmon_file)
+
+ cmd = f"ps -ef|grep -w nmon| grep -v grep | awk '{{print $2}}'"
+ try:
+ time.sleep(10)
+ _ = subprocess.check_output(cmd,shell=True).decode("utf-8")
+ except BaseException as e:
+ raise e
+
+ def td4153query(self):
+ tdLog.printNoPrefix("========== start to query operation ==========")
+
+ sqls = {
+ "select_all": "select * from stb1",
+ "select_join": "select * from t10, t11 where t10.ts=t11.ts"
+ }
+ for type, sql in sqls.items():
+ result_file = f"/tmp/queryResult_{type}.log"
+ query_file = self.createqueryfile(sql)
+ try:
+ self.taosdemotable(query_file, resultfile=result_file)
+ except subprocess.CalledProcessError as e:
+ out_put = e.output
+ if result_file:
+ print(f"execute rows {type.split('_')[1]} sql, the sql is: {sql}")
+ max_sql_time_cmd = f'''
+ grep -o Spent.*s {result_file} |awk 'NR==1{{max=$2;next}}{{max=max>$2?max:$2}}END{{print "Max=",max,"s"}}'
+ '''
+ max_sql_time = subprocess.check_output(max_sql_time_cmd, shell=True).decode("UTF-8")
+ print(f"{type.split('_')[1]} rows sql time : {max_sql_time}")
+
+ min_sql_time_cmd = f'''
+ grep -o Spent.*s {result_file} |awk 'NR==1{{min=$2;next}}{{min=min<$2?min:$2}}END{{print "Min=",min,"s"}}'
+ '''
+ min_sql_time = subprocess.check_output(min_sql_time_cmd, shell=True).decode("UTF-8")
+ print(f"{type.split('_')[1]} rows sql time : {min_sql_time}")
+
+ avg_sql_time_cmd = f'''
+ grep -o Spent.*s {result_file} |awk '{{sum+=$2}}END{{print "Average=",sum/NR,"s"}}'
+ '''
+ avg_sql_time = subprocess.check_output(avg_sql_time_cmd, shell=True).decode("UTF-8")
+ print(f"{type.split('_')[1]} rows sql time : {avg_sql_time}")
+
+ self.droptmpfile(query_file)
+ self.droptmpfile(result_file)
+
+ drop_query_tmt_file_cmd = " find ./ -name 'querySystemInfo-*' -type f -exec rm {} \; "
+ try:
+ _ = subprocess.check_output(drop_query_tmt_file_cmd, shell=True).decode("utf-8")
+ except subprocess.CalledProcessError as e:
+ _ = e.output
+ pass
+
+ def td4153(self):
+ self.td4153insert()
+ self.td4153query()
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '-r',
+ '--remove-cache',
+ action='store_true',
+ default=False,
+ help='clear cache before query (default: False)')
+ parser.add_argument(
+ '-d',
+ '--database-name',
+ action='store',
+ default='db',
+ type=str,
+ help='Database name to be created (default: db)')
+ parser.add_argument(
+ '-k',
+ '--keep-time',
+ action='store',
+ default=3650,
+ type=int,
+ help='Database keep parameters (default: 3650)')
+
+ args = parser.parse_args()
+ taosdemo = Taosdemo(args.remove_cache, args.database_name, args.keep_time)
+ # taosdemo.conn = taos.connect(
+ # taosdemo.host,
+ # taosdemo.user,
+ # taosdemo.password,
+ # taosdemo.config
+ # )
+
+ debugdir = taosdemo.getExeToolsDir()
+ cfgdir = taosdemo.getCfgDir()
+ cmd = f"{debugdir}/taosd -c {cfgdir} >/dev/null 2>&1 &"
+ try:
+ _ = subprocess.check_output(cmd, shell=True).decode("utf-8")
+ except subprocess.CalledProcessError as e:
+ _ = e.output
+
+ if taosdemo.clearCache:
+ # must be root permission
+ subprocess.check_output("echo 3 > /proc/sys/vm/drop_caches", shell=True).decode("utf-8")
+
+ taosdemo.td4153()
diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py
index a32cba167ef83a0036683f28297ca15d53939514..a45393e22284d675584c5dddd71fc507bcb2563f 100644
--- a/tests/pytest/tools/taosdemoPerformance.py
+++ b/tests/pytest/tools/taosdemoPerformance.py
@@ -24,7 +24,7 @@ class taosdemoPerformace:
self.host = "127.0.0.1"
self.user = "root"
self.password = "taosdata"
- self.config = "/etc/taosperf"
+ self.config = "/etc/perf"
self.conn = taos.connect(
self.host,
self.user,
@@ -77,7 +77,7 @@ class taosdemoPerformace:
insert_data = {
"filetype": "insert",
- "cfgdir": "/etc/taosperf",
+ "cfgdir": "/etc/perf",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
diff --git a/tests/script/general/parser/select_with_tags.sim b/tests/script/general/parser/select_with_tags.sim
index 7e5f31f7594bdde5e068e0d754ae863662696759..9d5214bfaaa6fdca44def066a85f49b87731c836 100644
--- a/tests/script/general/parser/select_with_tags.sim
+++ b/tests/script/general/parser/select_with_tags.sim
@@ -887,10 +887,16 @@ sql_error select tbname, t1 from select_tags_mt0 interval(1y);
#valid sql: select first(c1), last(c2), count(*) from select_tags_mt0 group by tbname, t1;
#valid sql: select first(c1), tbname, t1 from select_tags_mt0 group by t2;
+print ==================================>TD-4231
+sql_error select t1,tbname from select_tags_mt0 where c1<0
+sql_error select t1,tbname from select_tags_mt0 where c1<0 and tbname in ('select_tags_tb12')
+
+sql select tbname from select_tags_mt0 where tbname in ('select_tags_tb12');
+
sql_error select first(c1), last(c2), t1 from select_tags_mt0 group by tbname;
sql_error select first(c1), last(c2), tbname, t2 from select_tags_mt0 group by tbname;
sql_error select first(c1), count(*), t2, t1, tbname from select_tags_mt0 group by tbname;
-# this sql is valid: select first(c1), t2 from select_tags_mt0 group by tbname;
+#valid sql: select first(c1), t2 from select_tags_mt0 group by tbname;
#sql select first(ts), tbname from select_tags_mt0 group by tbname;
#sql select count(c1) from select_tags_mt0 where c1=99 group by tbname;
diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim
index 8e15c4f527db044d52cd725714f15b98161da2ce..b9ee508f78602cee7d6c9f17dbc3e250e7014f72 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim
@@ -158,7 +158,7 @@ if $dnode4Vtatus != offline then
sleep 2000
goto wait_dnode4_vgroup_offline
endi
-if $dnode3Vtatus != master then
+if $dnode3Vtatus != unsynced then
sleep 2000
goto wait_dnode4_vgroup_offline
endi