Merge branch '3.0' into fix/TD-20564

docs/examples/python/tmq_example.py

@@ -4,6 +4,7 @@ from taos.tmq import *
 conn = taos.connect()
 
 print("init")
+conn.execute("drop topic if exists topic_ctb_column")
 conn.execute("drop database if exists py_tmq")
 conn.execute("create database if not exists py_tmq vgroups 2")
 conn.select_db("py_tmq")
@@ -15,7 +16,6 @@ conn.execute("create table if not exists tb2 using stb1 tags(2)")
 conn.execute("create table if not exists tb3 using stb1 tags(3)")
 
 print("create topic")
-conn.execute("drop topic if exists topic_ctb_column")
 conn.execute(
     "create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1"
 )

source/libs/parser/src/parInsertSql.c

@@ -172,8 +172,8 @@ static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModifO
 }
 
 // pStmt->pSql -> field1_name, ...)
-static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, SParsedDataColInfo* pColList,
-                                 SSchema* pSchema) {
+static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, bool isTags,
+                                 SParsedDataColInfo* pColList, SSchema* pSchema) {
   col_id_t nCols = pColList->numOfCols;
 
   pColList->numOfBound = 0;
@@ -227,6 +227,10 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, S
     }
   }
 
+  if (!isTags && pColList->cols[0].valStat == VAL_STAT_NONE) {
+    return buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null");
+  }
+
   pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED;
 
   if (!isOrdered) {
@@ -525,7 +529,7 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt
   }
 
   pStmt->pSql += index;
-  return parseBoundColumns(pCxt, &pStmt->pSql, &pCxt->tags, pTagsSchema);
+  return parseBoundColumns(pCxt, &pStmt->pSql, true, &pCxt->tags, pTagsSchema);
 }
 
 static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken,
@@ -792,6 +796,8 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isSt
       *pMissCache = true;
     } else if (isStb && TSDB_SUPER_TABLE != (*pTableMeta)->tableType) {
       code = buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed");
+    } else if (!isStb && TSDB_SUPER_TABLE == (*pTableMeta)->tableType) {
+      code = buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported");
     }
   }
   return code;
@@ -935,11 +941,12 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpS
       return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z);
     }
     // pStmt->pSql -> field1_name, ...)
-    return parseBoundColumns(pCxt, &pStmt->pSql, &pDataBuf->boundColumnInfo, getTableColumnSchema(pStmt->pTableMeta));
+    return parseBoundColumns(pCxt, &pStmt->pSql, false, &pDataBuf->boundColumnInfo,
+                             getTableColumnSchema(pStmt->pTableMeta));
   }
 
   if (NULL != pStmt->pBoundCols) {
-    return parseBoundColumns(pCxt, &pStmt->pBoundCols, &pDataBuf->boundColumnInfo,
+    return parseBoundColumns(pCxt, &pStmt->pBoundCols, false, &pDataBuf->boundColumnInfo,
                              getTableColumnSchema(pStmt->pTableMeta));
   }
 
@@ -1571,16 +1578,16 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt
 
 static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); }
 
-static int32_t createVnodeModifOpStmt(SParseContext* pCxt, bool reentry, SNode** pOutput) {
+static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, SNode** pOutput) {
   SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT);
   if (NULL == pStmt) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  if (pCxt->pStmtCb) {
+  if (pCxt->pComCxt->pStmtCb) {
     TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT);
   }
-  pStmt->pSql = pCxt->pSql;
+  pStmt->pSql = pCxt->pComCxt->pSql;
   pStmt->freeHashFunc = insDestroyBlockHashmap;
   pStmt->freeArrayFunc = insDestroyBlockArrayList;
 
@@ -1604,7 +1611,7 @@ static int32_t createVnodeModifOpStmt(SParseContext* pCxt, bool reentry, SNode**
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t createInsertQuery(SParseContext* pCxt, SQuery** pOutput) {
+static int32_t createInsertQuery(SInsertParseContext* pCxt, SQuery** pOutput) {
   SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
   if (NULL == pQuery) {
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -1667,11 +1674,15 @@ static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifOpSt
                      sizeof(SVgroupInfo));
 }
 
-static int32_t getTableSchemaFromMetaData(const SMetaData* pMetaData, SVnodeModifOpStmt* pStmt, bool isStb) {
+static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMetaData* pMetaData,
+                                          SVnodeModifOpStmt* pStmt, bool isStb) {
   int32_t code = checkAuthFromMetaData(pMetaData->pUser);
   if (TSDB_CODE_SUCCESS == code) {
     code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta);
   }
+  if (TSDB_CODE_SUCCESS == code && !isStb && TSDB_SUPER_TABLE == pStmt->pTableMeta->tableType) {
+    code = buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported");
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = getTableVgroupFromMetaData(pMetaData->pTableHash, pStmt, isStb);
   }
@@ -1696,24 +1707,25 @@ static void clearCatalogReq(SCatalogReq* pCatalogReq) {
   pCatalogReq->pUser = NULL;
 }
 
-static int32_t setVnodeModifOpStmt(SParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData,
+static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData,
                                    SVnodeModifOpStmt* pStmt) {
   clearCatalogReq(pCatalogReq);
 
   if (pStmt->usingTableProcessing) {
-    return getTableSchemaFromMetaData(pMetaData, pStmt, true);
+    return getTableSchemaFromMetaData(pCxt, pMetaData, pStmt, true);
   }
-  return getTableSchemaFromMetaData(pMetaData, pStmt, false);
+  return getTableSchemaFromMetaData(pCxt, pMetaData, pStmt, false);
 }
 
-static int32_t resetVnodeModifOpStmt(SParseContext* pCxt, SQuery* pQuery) {
+static int32_t resetVnodeModifOpStmt(SInsertParseContext* pCxt, SQuery* pQuery) {
   nodesDestroyNode(pQuery->pRoot);
 
   int32_t code = createVnodeModifOpStmt(pCxt, true, &pQuery->pRoot);
   if (TSDB_CODE_SUCCESS == code) {
     SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot;
 
-    (*pCxt->pStmtCb->getExecInfoFn)(pCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj, &pStmt->pTableBlockHashObj);
+    (*pCxt->pComCxt->pStmtCb->getExecInfoFn)(pCxt->pComCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj,
+                                             &pStmt->pTableBlockHashObj);
     if (NULL == pStmt->pVgroupsHashObj) {
       pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
     }
@@ -1729,13 +1741,13 @@ static int32_t resetVnodeModifOpStmt(SParseContext* pCxt, SQuery* pQuery) {
   return code;
 }
 
-static int32_t initInsertQuery(SParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData,
+static int32_t initInsertQuery(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData,
                                SQuery** pQuery) {
   if (NULL == *pQuery) {
     return createInsertQuery(pCxt, pQuery);
   }
 
-  if (NULL != pCxt->pStmtCb) {
+  if (NULL != pCxt->pComCxt->pStmtCb) {
     return resetVnodeModifOpStmt(pCxt, *pQuery);
   }
 
@@ -1896,7 +1908,7 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal
       .usingDuplicateTable = false,
   };
 
-  int32_t code = initInsertQuery(pCxt, pCatalogReq, pMetaData, pQuery);
+  int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery);
   if (TSDB_CODE_SUCCESS == code) {
     code = parseInsertSqlImpl(&context, (SVnodeModifOpStmt*)(*pQuery)->pRoot);
   }

source/libs/sync/src/syncCommit.c

@@ -171,7 +171,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
     pSyncNode->pLogStore->syncLogUpdateCommitIndex(pSyncNode->pLogStore, pSyncNode->commitIndex);
 
     // execute fsm
-    if (pSyncNode->pFsm != NULL) {
+    if (pSyncNode != NULL && pSyncNode->pFsm != NULL) {
       int32_t code = syncNodeDoCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state);
       if (code != 0) {
         sNError(pSyncNode, "advance commit index error, do commit begin:%" PRId64 ", end:%" PRId64, beginIndex,

source/libs/sync/src/syncRaftEntry.c

@@ -64,7 +64,7 @@ SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, Syn
 }
 
 SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg) {
-  SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen);
+  SSyncRaftEntry* pEntry = syncEntryBuild((int32_t)(pMsg->dataLen));
   if (pEntry == NULL) return NULL;
 
   memcpy(pEntry, pMsg->data, pMsg->dataLen);
@@ -91,15 +91,14 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId)
 
 void syncEntryDestory(SSyncRaftEntry* pEntry) {
   if (pEntry != NULL) {
-    taosMemoryFree(pEntry);
-
     sTrace("free entry: %p", pEntry);
+    taosMemoryFree(pEntry);
   }
 }
 
 void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg) {
   pRpcMsg->msgType = pEntry->originalRpcType;
-  pRpcMsg->contLen = pEntry->dataLen;
+  pRpcMsg->contLen = (int32_t)(pEntry->dataLen);
   pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
   memcpy(pRpcMsg->pCont, pEntry->data, pRpcMsg->contLen);
 }
@@ -339,7 +338,8 @@ int32_t raftEntryCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index,
   SSyncRaftEntry* pEntry = NULL;
   int32_t         code = raftEntryCacheGetEntryP(pCache, index, &pEntry);
   if (code == 1) {
-    *ppEntry = taosMemoryMalloc((int64_t)(pEntry->bytes));
+    int32_t bytes = (int32_t)pEntry->bytes;
+    *ppEntry = taosMemoryMalloc((int64_t)bytes);
     memcpy(*ppEntry, pEntry, pEntry->bytes);
     (*ppEntry)->rid = -1;
   } else {

source/libs/sync/src/syncReplication.c

@@ -91,7 +91,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapsh
   if (code == 0) {
     ASSERT(pEntry != NULL);
 
-    code = syncBuildAppendEntries(&rpcMsg, pEntry->bytes, pSyncNode->vgId);
+    code = syncBuildAppendEntries(&rpcMsg, (int32_t)(pEntry->bytes), pSyncNode->vgId);
     ASSERT(code == 0);
 
     pMsg = rpcMsg.pCont;

source/libs/sync/src/syncUtil.c

@@ -229,7 +229,10 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo
   int32_t writeLen = vsnprintf(eventLog, sizeof(eventLog), format, argpointer);
   va_end(argpointer);
 
-  int32_t aqItems = pNode->pFsm->FpApplyQueueItems(pNode->pFsm);
+  int32_t aqItems = 0;
+  if (pNode != NULL && pNode->pFsm != NULL && pNode->pFsm->FpApplyQueueItems != NULL) {
+    aqItems = pNode->pFsm->FpApplyQueueItems(pNode->pFsm);
+  }
 
   // restore error code
   terrno = errCode;

tests/script/runAllSimCases.sh

@@ -11,6 +11,9 @@ set -e
 VALGRIND=0
 LOG_BK_DIR=/data/valgrind_log_backup     # 192.168.0.203
 SIM_FILES=./jenkins/basic.txt
+cases_task_file=../parallel_test/cases.task
+
+cat $cases_task_file | grep "./test.sh " | awk -F, '{print $5}' > ./jenkins/basic.txt
 
 while getopts "v:r:f:" arg
 do

tests/script/sh/checkAsan.sh

@@ -21,15 +21,24 @@ LOG_DIR=$TAOS_DIR/sim/asan
 error_num=`cat ${LOG_DIR}/*.asan | grep "ERROR" | wc -l`
 memory_leak=`cat ${LOG_DIR}/*.asan | grep "Direct leak" | wc -l`
 indirect_leak=`cat ${LOG_DIR}/*.asan | grep "Indirect leak" | wc -l`
+python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l`
 
 # ignore
+
+# TD-20368
+# /root/TDengine/contrib/zlib/trees.c:873:5: runtime error: null pointer passed as argument 2, which is declared to never be null
+
+# TD-20494 TD-20452
 # /root/TDengine/source/libs/scalar/src/sclfunc.c:735:11: runtime error: 4.75783e+11 is outside the range of representable values of type 'signed char'
 # /root/TDengine/source/libs/scalar/src/sclfunc.c:790:11: runtime error: 3.4e+38 is outside the range of representable values of type 'long int'
 # /root/TDengine/source/libs/scalar/src/sclfunc.c:772:11: runtime error: 3.52344e+09 is outside the range of representable values of type 'int'
 # /root/TDengine/source/libs/scalar/src/sclfunc.c:753:11: runtime error: 4.75783e+11 is outside the range of representable values of type 'short int'
-runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type" | wc -l`
 
-python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l`
+# TD-20569  
+# /root/TDengine/source/libs/function/src/builtinsimpl.c:856:29: runtime error: signed integer overflow: 9223372036854775806 + 9223372036854775805 cannot be represented in type 'long int'
+# /root/TDengine/source/libs/scalar/src/sclvector.c:1075:66: runtime error: signed integer overflow: 9223372034707292160 + 1668838476672 cannot be represented in type 'long int'
+
+runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "builtinsimpl.c.*signed integer overflow"| grep -v "sclvector.c.*signed integer overflow" | wc -l`
 
 echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m"
 echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m"

tests/script/sh/stop_dnodes.sh

@@ -26,3 +26,17 @@ while [ -n "$PID" ]; do
   fi
   PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
 done
+
+PID=`ps -ef|grep -w taos | grep -v grep | awk '{print $2}'`
+while [ -n "$PID" ]; do
+  echo kill -9 $PID
+  #pkill -9 taosd
+  kill -9 $PID
+  echo "Killing taosd processes"
+  if [ "$OS_TYPE" != "Darwin" ]; then
+    fuser -k -n tcp 6030
+  else
+    lsof -nti:6030 | xargs kill -9
+  fi
+  PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+done

tests/system-test/0-others/taosShellNetChk.py

@@ -230,9 +230,11 @@ class TDTestCase:
                 tdLog.exit('taos -n client fail!')
         finally:
             if platform.system().lower() == 'windows':
-                os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9')
+                tdLog.info("ps -a | grep taos | awk \'{print $2}\' | xargs kill -9")
+                # os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9')
             else:
-                os.system('pkill -9 taos')
+                tdLog.info("pkill -9 taos")
+                # os.system('pkill -9 taos')
 
     def stop(self):
         tdSql.close()