Merge branch 'develop' into hotfix/crash

src/client/src/tscServer.c

@@ -209,6 +209,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
     tscError("%p sql is already released", pSql->signature);
     return;
   }
+
   if (pSql->signature != pSql) {
     tscError("%p sql is already released, signature:%p", pSql, pSql->signature);
     return;
@@ -217,10 +218,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
   STscObj *pObj = pSql->pTscObj;
-  // tscTrace("%p msg:%s is received from server", pSql, taosMsg[rpcMsg->msgType]);
 
-  if (pObj->signature != pObj) {
-    tscTrace("%p sql is already released or DB connection is closed, freed:%d pObj:%p signature:%p", pSql, pSql->freed,
+  if (pObj->signature != pObj || pSql->freed == 1) {
+    tscTrace("%p sqlObj needs to be released or DB connection is closed, freed:%d pObj:%p signature:%p", pSql, pSql->freed,
              pObj, pObj->signature);
     tscFreeSqlObj(pSql);
     rpcFreeCont(rpcMsg->pCont);
@@ -375,7 +375,7 @@ int tscProcessSql(SSqlObj *pSql) {
   
   SQueryInfo *    pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   STableMetaInfo *pTableMetaInfo = NULL;
-  uint16_t        type = 0;
+  uint32_t        type = 0;
 
   if (pQueryInfo != NULL) {
     pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
@@ -424,7 +424,7 @@ void tscKillSTableQuery(SSqlObj *pSql) {
      * sub-queries not correctly released and master sql object of super table query reaches an abnormal state.
      */
     pSql->pSubs[i]->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
-    //taosStopRpcConn(pSql->pSubs[i]->thandle);
+//    taosStopRpcConn(pSql->pSubs[i]->);
   }
 
   /*

src/client/src/tscSql.c

@@ -219,6 +219,11 @@ void waitForQueryRsp(void *param, TAOS_RES *tres, int code) {
   sem_post(&pSql->rspSem);
 }
 
+static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) {
+  SSqlObj* pSql = (SSqlObj*) tres;
+  sem_post(&pSql->rspSem);
+}
+
 TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) {
   STscObj *pObj = (STscObj *)taos;
   if (pObj == NULL || pObj->signature != pObj) {
@@ -369,11 +374,6 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) {
   return (pQueryInfo->order.order == TSDB_ORDER_DESC) ? pRes->numOfRows : -pRes->numOfRows;
 }
 
-static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) {
-  SSqlObj* pSql = (SSqlObj*) tres;
-  sem_post(&pSql->rspSem);
-}
-
 TAOS_ROW taos_fetch_row(TAOS_RES *res) {
   SSqlObj *pSql = (SSqlObj *)res;
   if (pSql == NULL || pSql->signature != pSql) {
@@ -476,7 +476,7 @@ int taos_select_db(TAOS *taos, const char *db) {
 }
 
 // send free message to vnode to free qhandle and corresponding resources in vnode
-static void tscFreeQhandleInVnode(SSqlObj* pSql) {
+static bool tscFreeQhandleInVnode(SSqlObj* pSql) {
   SSqlCmd* pCmd = &pSql->cmd;
   SSqlRes* pRes = &pSql->res;
 
@@ -496,10 +496,19 @@ static void tscFreeQhandleInVnode(SSqlObj* pSql) {
     tscProcessSql(pSql);
 
     // in case of sync model query, waits for response and then goes on
-    if (pSql->fp == waitForQueryRsp || pSql->fp == waitForRetrieveRsp) {
-      sem_wait(&pSql->rspSem);
-    }
+//    if (pSql->fp == waitForQueryRsp || pSql->fp == waitForRetrieveRsp) {
+//      sem_wait(&pSql->rspSem);
+
+//      tscFreeSqlObj(pSql);
+//      tscTrace("%p sqlObj is freed by app", pSql);
+//    } else {
+      tscTrace("%p sqlObj will be freed while rsp received", pSql);
+//    }
+
+    return true;
   }
+
+  return false;
 }
 
 void taos_free_result(TAOS_RES *res) {
@@ -527,10 +536,10 @@ void taos_free_result(TAOS_RES *res) {
   }
 
   pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
-  tscFreeQhandleInVnode(pSql);
+  if (!tscFreeQhandleInVnode(pSql)) {
     tscFreeSqlObj(pSql);
-
-  tscTrace("%p sql result is freed by app", pSql);
+    tscTrace("%p sqlObj is freed by app", pSql);
+  }
 }
 
 // todo should not be used in async query

src/client/src/tscUtil.c

@@ -134,24 +134,6 @@ void tscGetDBInfoFromMeterId(char* tableId, char* db) {
   db[0] = 0;
 }
 
-//STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) {
-//  if (pSidList == NULL) {
-//    tscError("illegal sidlist");
-//    return 0;
-//  }
-//
-//  if (idx < 0 || idx >= pSidList->numOfSids) {
-//    int32_t sidRange = (pSidList->numOfSids > 0) ? (pSidList->numOfSids - 1) : 0;
-//
-//    tscError("illegal sidIdx:%d, reset to 0, sidIdx range:%d-%d", idx, 0, sidRange);
-//    idx = 0;
-//  }
-//
-//  assert(pSidList->pSidExtInfoList[idx] >= 0);
-//
-//  return (STableIdInfo*)(pSidList->pSidExtInfoList[idx] + (char*)pSidList);
-//}
-
 bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) {
   if (pQueryInfo == NULL) {
     return false;
@@ -176,8 +158,7 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) {
     return false;
   }
 
-  if (((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) != TSDB_QUERY_TYPE_STABLE_SUBQUERY) &&
-      pQueryInfo->command == TSDB_SQL_SELECT) {
+  if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->command == TSDB_SQL_SELECT) {
     return UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
   }
 

src/common/src/tglobal.c

@@ -1262,9 +1262,10 @@ bool taosCheckGlobalCfg() {
     taosSetAllDebugFlag();
   }
   
-  if (tsLocalFqdn[0] == 0) 
-    taosGetFqdn(tsLocalEp);
-  else 
+  if (tsLocalFqdn[0] == 0) {
+    taosGetFqdn(tsLocalFqdn);
+  }
+
   strcpy(tsLocalEp, tsLocalFqdn);
 
   snprintf(tsLocalEp + strlen(tsLocalEp), sizeof(tsLocalEp), ":%d", tsServerPort);

src/inc/trpc.h

@@ -83,7 +83,7 @@ void  rpcSendResponse(const SRpcMsg *pMsg);
 void  rpcSendRedirectRsp(void *pConn, const SRpcIpSet *pIpSet); 
 int   rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
 void  rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pReq, SRpcMsg *pRsp);
-void  rpcReportProgress(void *pConn, char *pCont, int contLen);
+int   rpcReportProgress(void *pConn, char *pCont, int contLen);
 
 #ifdef __cplusplus
 }

src/mnode/src/mnodeSdb.c

@@ -451,7 +451,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
 
   pthread_mutex_unlock(&pTable->mutex);
 
-  sdbTrace("table:%s, insert record:%s to hash, rowSize:%d vnumOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName,
+  sdbTrace("table:%s, insert record:%s to hash, rowSize:%d numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName,
            sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, sdbGetVersion());
 
   (*pTable->insertFp)(pOper);
@@ -475,7 +475,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
   pTable->numOfRows--;
   pthread_mutex_unlock(&pTable->mutex);
 
-  sdbTrace("table:%s, delete record:%s from hash, numOfRows:%d" PRId64 "version:%" PRIu64, pTable->tableName,
+  sdbTrace("table:%s, delete record:%s from hash, numOfRows:%" PRId64 "version:%" PRIu64, pTable->tableName,
            sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
 
   int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1;

src/query/src/qExecutor.c

@@ -4470,7 +4470,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
 
   // query error occurred or query is killed, abort current execution
   if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) {
-    qTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code);
+    qTrace("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code));
     return;
   }
 
@@ -4491,7 +4491,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
   setQueryStatus(pQuery, QUERY_COMPLETED);
 
   if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) {
-    qTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code);
+    qTrace("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code));
     return;
   }
 
@@ -5851,6 +5851,8 @@ void qDestroyQueryInfo(qinfo_t qHandle) {
   }
 
   int16_t ref = T_REF_DEC(pQInfo);
+  qTrace("QInfo:%p dec refCount, value:%d", pQInfo, ref);
+
   if (ref == 0) {
     doDestoryQueryInfo(pQInfo);
   }

src/rpc/src/rpcMain.c

@@ -393,7 +393,6 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
   if ( pConn->inType == 0 || pConn->user[0] == 0 ) {
     tTrace("%s, connection is already released, rsp wont be sent", pConn->info);
     rpcUnlockConn(pConn);
-    rpcDecRef(pRpc);
     return;
   }
 
@@ -487,12 +486,19 @@ void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pMsg, SRpcMsg
 }
 
 // this API is used by server app to keep an APP context in case connection is broken
-void rpcReportProgress(void *handle, char *pCont, int contLen) {
+int rpcReportProgress(void *handle, char *pCont, int contLen) {
   SRpcConn *pConn = (SRpcConn *)handle;
 
+  if (pConn->user[0]) {
     // pReqMsg and reqMsgLen is re-used to store the context from app server
     pConn->pReqMsg = pCont;     
     pConn->reqMsgLen = contLen;
+    return 0;
+  } 
+
+  tTrace("%s, rpc connection is already released", pConn->info);
+  rpcFreeCont(pCont);
+  return -1;
 }
 
 static void rpcFreeMsg(void *msg) {

src/vnode/src/vnodeRead.c

@@ -68,6 +68,7 @@ static void vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId)
   killQueryMsg->header.vgId = htonl(vgId);
   killQueryMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg));
 
+  vTrace("QInfo:%p register qhandle to connect:%p", qhandle, handle);
   rpcReportProgress(handle, (char*) killQueryMsg, sizeof(SRetrieveTableMsg));
 }
 
@@ -85,10 +86,11 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
     killQueryMsg->free = htons(killQueryMsg->free);
     killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle);
 
+    vWarn("QInfo:%p connection %p broken, kill query", killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
     assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1);
-    qDestroyQueryInfo((qinfo_t) killQueryMsg->qhandle);
 
-    return TSDB_CODE_SUCCESS;
+    qKillQuery((qinfo_t) killQueryMsg->qhandle);
+    return TSDB_CODE_TSC_QUERY_CANCELLED; // todo change the error code
   }
 
   int32_t code = TSDB_CODE_SUCCESS;

tests/script/unique/arbitrator/dn3_mn1_replica_change.sim

@@ -51,7 +51,7 @@ system sh/exec.sh -n dnode2 -s start
 sql create dnode $hostname2
 sleep 3000
 
-$totalTableNum = 10000
+$totalTableNum = 10
 $sleepTimer = 10000
 
 $db = db
@@ -192,7 +192,7 @@ if $data00 != $totalRows then
 endi
 
 
-print ============== step4: stop dnode2 for checking if sync success
+print ============== step4: stop dnode2 for checking if sync ok
 system sh/exec.sh -n dnode2 -s stop
 sleep $sleepTimer
 

tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim

@@ -55,7 +55,7 @@ sql create dnode $hostname3
 sql create dnode $hostname4
 sleep 3000
 
-$totalTableNum = 1000
+$totalTableNum = 10
 $sleepTimer = 10000
 
 $db = db

tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim

@@ -173,7 +173,7 @@ if $rows != 3 then
   return -1
 endi
 
-print ============== step5: remove dnode4 director, then recreate dnode4 into cluster, result should success
+print ============== step5: remove dnode4 director, then recreate dnode4 into cluster, result should ok 
 system sh/exec.sh -n dnode4 -s stop
 system rm -rf ../../../sim/dnode4
 

tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim

@@ -55,7 +55,7 @@ sql create dnode $hostname3
 sql create dnode $hostname4
 sleep 3000
 
-$totalTableNum = 100
+$totalTableNum = 10
 $sleepTimer = 3000
 
 $db = db

tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim

@@ -55,7 +55,7 @@ sql create dnode $hostname3
 #sql create dnode $hostname4
 sleep 3000
 
-$totalTableNum = 100
+$totalTableNum = 10
 $sleepTimer = 3000
 
 $db = db
@@ -177,7 +177,7 @@ if $data00 != $totalRows then
 	return -1
 endi
 
-print ============== step5: stop dnode2, and check if dnode3 sync success
+print ============== step5: stop dnode2, and check if dnode3 sync ok 
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 sleep $sleepTimer
 wait_dnode2_offline_0:

tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim

@@ -55,7 +55,7 @@ sql create dnode $hostname3
 #sql create dnode $hostname4
 sleep 3000
 
-$totalTableNum = 100
+$totalTableNum = 10
 $sleepTimer = 3000
 
 $db = db
@@ -105,7 +105,7 @@ if $data00 != $totalRows then
 	return -1
 endi
 
-print ============== step5: stop dnode2, and check if dnode3 sync success
+print ============== step5: stop dnode2, and check if dnode3 sync ok 
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 sleep $sleepTimer
 wait_dnode2_offline_0:

tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim

@@ -55,7 +55,7 @@ sql create dnode $hostname3
 sql create dnode $hostname4
 sleep 3000
 
-$totalTableNum = 100
+$totalTableNum = 10
 $sleepTimer = 3000
 
 $db = db

tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim

@@ -55,7 +55,7 @@ sql create dnode $hostname3
 sql create dnode $hostname4
 sleep 3000
 
-$totalTableNum = 100
+$totalTableNum = 10
 $sleepTimer = 3000
 
 $db = db
@@ -152,7 +152,7 @@ print ============== step4: restart dnode2, then create database with replica 2,
 system sh/exec.sh -n dnode2 -s start
 sleep 3000
 
-$totalTableNum = 100
+$totalTableNum = 10
 $sleepTimer = 3000
 
 $db = db1

tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim

@@ -56,7 +56,7 @@ sql create dnode $hostname2
 sql create dnode $hostname4
 sleep 3000
 
-$totalTableNum = 100
+$totalTableNum = 10
 $sleepTimer = 3000
 
 $db = db

tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim

@@ -67,7 +67,7 @@ sql use $db
 
 # create table , insert data
 $stb = stb
-sql create table $stb (ts timestamp, c1 int, c2 int) tags(t1 int)
+sql create table $stb (ts timestamp, c1 int, c2 int) tags(t0 int, t1 int)
 $rowNum = 500
 $tblNum = $totalTableNum
 $totalRows = 0
@@ -77,7 +77,7 @@ $tsEnd = 0
 $i = 0
 while $i < $tblNum
   $tb = tb . $i
-  sql create table $tb using $stb tags( $i )
+  sql create table $tb using $stb tags( $i , $i )
  
   $x = 0
   while $x < $rowNum
@@ -164,7 +164,7 @@ $totalRows = 0
 $i = 0
 while $i < $tblNum
   $tb = tb . $i
-  sql create table $tb using $stb tags( $i , $i )
+  sql create table $tb using $stb tags( $i , $i , $i )
  
   $x = 0
   while $x < $rowNum

tests/script/unique/arbitrator/offline_replica2_createTable_online.sim

@@ -56,7 +56,7 @@ sql create dnode $hostname2
 sql create dnode $hostname4
 sleep 3000
 
-$totalTableNum = 100
+$totalTableNum = 10
 $sleepTimer = 3000
 
 $maxTables = $totalTableNum * 2

tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim

@@ -56,7 +56,7 @@ sql create dnode $hostname2
 sql create dnode $hostname4
 sleep 3000
 
-$totalTableNum = 100
+$totalTableNum = 10
 $sleepTimer = 3000
 
 $db = db

tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim

@@ -56,7 +56,7 @@ sql create dnode $hostname2
 sql create dnode $hostname4
 sleep 3000
 
-$totalTableNum = 100
+$totalTableNum = 10
 $sleepTimer = 3000
 
 $db = db
@@ -146,14 +146,14 @@ endi
 
 print ============== step4: drop some tables
 $i = 1
-$dropTblNum = 21
+$dropTblNum = 6
 
 while $i < $dropTblNum
   $tb = tb . $i
   sql drop table if exists $tb
   $i = $i + 1
 endw 
-$tblNum = $tblNum - 20
+$tblNum = $tblNum - 5
 
 print ============== step5: restart dnode4, waiting dnode4 synced
 system sh/exec.sh -n dnode4 -s start

tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim

@@ -56,7 +56,7 @@ sql create dnode $hostname3
 sql create dnode $hostname4
 sleep 3000
 
-$totalTableNum = 100
+$totalTableNum = 10
 $sleepTimer = 3000
 
 $db = db

tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim

@@ -67,7 +67,7 @@ sql use $db
 
 # create table , insert data
 $stb = stb
-sql create table $stb (ts timestamp, c1 int, c2 int) tags(t1 int)
+sql create table $stb (ts timestamp, c1 int, c2 int) tags(t0 int, t1 int)
 $rowNum = 500
 $tblNum = $totalTableNum
 $totalRows = 0
@@ -77,7 +77,7 @@ $tsEnd = 0
 $i = 0
 while $i < $tblNum
   $tb = tb . $i
-  sql create table $tb using $stb tags( $i )
+  sql create table $tb using $stb tags( $i , $i )
  
   $x = 0
   while $x < $rowNum
@@ -164,7 +164,7 @@ $totalRows = 0
 $i = 0
 while $i < $tblNum
   $tb = tb . $i
-  sql create table $tb using $stb tags( $i , $i )
+  sql create table $tb using $stb tags( $i , $i , $i )
  
   $x = 0
   while $x < $rowNum

tests/script/unique/arbitrator/offline_replica3_createTable_online.sim

@@ -56,7 +56,7 @@ sql create dnode $hostname3
 sql create dnode $hostname4
 sleep 3000
 
-$totalTableNum = 100
+$totalTableNum = 10
 $sleepTimer = 3000
 $maxTables = $totalTableNum * 2
 

tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim

@@ -56,7 +56,7 @@ sql create dnode $hostname3
 sql create dnode $hostname4
 sleep 3000
 
-$totalTableNum = 100
+$totalTableNum = 10
 $sleepTimer = 3000
 
 $db = db

tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim

@@ -56,7 +56,7 @@ sql create dnode $hostname3
 sql create dnode $hostname4
 sleep 3000
 
-$totalTableNum = 100
+$totalTableNum = 10
 $sleepTimer = 3000
 
 $db = db
@@ -146,14 +146,14 @@ endi
 
 print ============== step4: drop some tables
 $i = 1
-$dropTblNum = 21
+$dropTblNum = 6
 
 while $i < $dropTblNum
   $tb = tb . $i
   sql drop table if exists $tb
   $i = $i + 1
 endw 
-$tblNum = $tblNum - 20
+$tblNum = $tblNum - 5
 
 print ============== step5: restart dnode4, waiting dnode4 synced
 system sh/exec.sh -n dnode4 -s start