Merge branch 'develop' into feature/2.0tsdb

src/client/src/tscAsync.c

@@ -198,6 +198,10 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
 
+  // user-defined callback function is stored in fetchFp
+  pSql->fetchFp = fp;
+  pSql->fp = tscAsyncFetchRowsProxy;
+
   if (pRes->qhandle == 0) {
     tscError("qhandle is NULL");
     pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE;
@@ -205,10 +209,6 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi
     return;
   }
 
-  // user-defined callback function is stored in fetchFp
-  pSql->fetchFp = fp;
-  pSql->fp = tscAsyncFetchRowsProxy;
-
   pSql->param = param;
   tscResetForNextRetrieve(pRes);
   

src/mnode/src/mnodeProfile.c

@@ -115,7 +115,7 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po
   uint64_t expireTime = CONN_KEEP_TIME * 1000 + (uint64_t)taosGetTimestampMs();
   SConnObj *pConn = taosCacheUpdateExpireTimeByName(tsMnodeConnCache, &connId, sizeof(int32_t), expireTime);
   if (pConn == NULL) {
-    mError("connId:%d, is already destroyed, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port);
+    mDebug("connId:%d, is already destroyed, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port);
     return NULL;
   }
 

src/plugins/http/src/httpContext.c

@@ -137,7 +137,7 @@ void httpReleaseContext(HttpContext *pContext) {
   assert(refCount >= 0);
 
   HttpContext **ppContext = pContext->ppContext;
-  httpDebug("context:%p, is releasd, data:%p refCount:%d", pContext, ppContext, refCount);
+  httpDebug("context:%p, is released, data:%p refCount:%d", pContext, ppContext, refCount);
 
   if (tsHttpServer.contextCache != NULL) {
     taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false);

src/plugins/http/src/httpSql.c

@@ -47,6 +47,10 @@ void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numO
     }
   }
 
+  if (tscResultsetFetchCompleted(result)) {
+    isContinue = false;
+  }
+
   if (isContinue) {
     // retrieve next batch of rows
     httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, continue retrieve, numOfRows:%d, sql:%s",
@@ -75,7 +79,8 @@ void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) {
   HttpContext *pContext = (HttpContext *)param;
   if (pContext == NULL) return;
 
-  HttpSqlCmds *     multiCmds = pContext->multiCmds;
+  code = taos_errno(result);  
+  HttpSqlCmds *multiCmds = pContext->multiCmds;
   HttpEncodeMethod *encode = pContext->encodeMethod;
 
   HttpSqlCmd *singleCmd = multiCmds->cmds + multiCmds->pos;
@@ -109,8 +114,8 @@ void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) {
     return;
   }
 
-  int num_fields = taos_field_count(result);
-  if (num_fields == 0) {
+  bool isUpdate = tscIsUpdateQuery(result);
+  if (isUpdate) {
     // not select or show commands
     int affectRows = taos_affected_rows(result);
     httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, affect rows:%d, sql:%s",
@@ -238,6 +243,11 @@ void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int unUsedCode)
   if (pContext == NULL) return;
 
   int32_t code = taos_errno(result);
+  if (code != unUsedCode) {
+    httpError("context:%p, fd:%d, ip:%s, user:%s, resultset code:%s input code:%s not matched, sqlObj:%p", pContext,
+              pContext->fd, pContext->ipstr, pContext->user, tstrerror(code), tstrerror(unUsedCode), (SSqlObj *)result);
+  }
+
   HttpEncodeMethod *encode = pContext->encodeMethod;
 
   if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {

src/query/src/qExecutor.c

@@ -6333,6 +6333,7 @@ int32_t qKillQuery(qinfo_t qinfo) {
     return TSDB_CODE_QRY_INVALID_QHANDLE;
   }
 
+  sem_post(&pQInfo->dataReady);
   setQueryKilled(pQInfo);
   return TSDB_CODE_SUCCESS;
 }
@@ -6545,13 +6546,14 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) {
 
   SQueryMgmt *pQueryMgmt = pMgmt;
   if (pQueryMgmt->qinfoPool == NULL) {
+    qError("QInfo:%p failed to add qhandle into qMgmt, since qMgmt is closed", (void *)qInfo);
     return NULL;
   }
 
   pthread_mutex_lock(&pQueryMgmt->lock);
   if (pQueryMgmt->closed) {
     pthread_mutex_unlock(&pQueryMgmt->lock);
-
+    qError("QInfo:%p failed to add qhandle into cache, since qMgmt is colsing", (void *)qInfo);
     return NULL;
   } else {
     uint64_t handleVal = (uint64_t) qInfo;

src/util/src/tcache.c

@@ -294,7 +294,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v
     }
   } else {  // old data exists, update the node
     pNode = taosUpdateCacheImpl(pCacheObj, pOld, key, keyLen, pData, dataSize, duration * 1000L);
-    uDebug("cache:%s, key:%p, %p exist in cache, updated", pCacheObj->name, key, pNode->data);
+    uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode->data, pOld);
   }
 
   __cache_unlock(pCacheObj);
@@ -307,6 +307,8 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen
     return NULL;
   }
 
+  void *pData = NULL;
+
   __cache_rd_lock(pCacheObj);
 
   SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen);
@@ -314,19 +316,21 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen
   int32_t ref = 0;
   if (ptNode != NULL) {
     ref = T_REF_INC(*ptNode);
+    pData = (*ptNode)->data;
   }
+
   __cache_unlock(pCacheObj);
 
-  if (ptNode != NULL) {
+  if (pData != NULL) {
     atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1);
-    uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, (*ptNode)->data, ref);
+    uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData, ref);
   } else {
     atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1);
     uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key);
   }
 
   atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1);
-  return (ptNode != NULL) ? (*ptNode)->data : NULL;
+  return pData;
 }
 
 void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t keyLen, uint64_t expireTime) {
@@ -453,21 +457,20 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
   } else {
     uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, T_REF_VAL_GET(pNode) - 1);
 
+    __cache_wr_lock(pCacheObj);
+
     // NOTE: once refcount is decrease, pNode may be freed by other thread immediately.
     int32_t ref = T_REF_DEC(pNode);
 
-    if (inTrashCan) {
+    if (inTrashCan && (ref == 0)) {
       // Remove it if the ref count is 0.
       // The ref count does not need to load and check again after lock acquired, since ref count can not be increased when
       // the node is in trashcan.
-      if (ref == 0) {
-        __cache_wr_lock(pCacheObj);
       assert(pNode->pTNodeHeader->pData == pNode);
       taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader);
-        __cache_unlock(pCacheObj);
     }
 
-    }
+    __cache_unlock(pCacheObj);
   }
 
 //   else {

src/vnode/src/vnodeRead.c

@@ -108,9 +108,10 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
     if (code == TSDB_CODE_SUCCESS) {
       handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t) pQInfo);
       if (handle == NULL) {  // failed to register qhandle
+        vError("vgId:%d QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo,
+               tstrerror(pRsp->code));
         pRsp->code = TSDB_CODE_QRY_INVALID_QHANDLE;
         qDestroyQueryInfo(pQInfo);  // destroy it directly
-        vError("vgId:%d QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void*) pQInfo, tstrerror(pRsp->code));
       } else {
         assert(*handle == pQInfo);
         pRsp->qhandle = htobe64((uint64_t) pQInfo);

tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim

@@ -29,7 +29,7 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
 
-$totalTableNum = 10
+$totalTableNum = 40
 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $totalTableNum
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v $totalTableNum
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v $totalTableNum
@@ -65,9 +65,10 @@ sleep 3000
 $totalTableNum = 20
 $sleepTimer = 3000
 
+$maxTables = $totalTableNum * 2 
 $db = db
-print create database $db replica 3 maxTables $totalTableNum
-sql create database $db replica 3 maxTables $totalTableNum
+print create database $db replica 3 maxTables $maxTables
+sql create database $db replica 3 maxTables $maxTables
 sql use $db
 
 # create table , insert data

tests/script/unique/cluster/client5.sim

 $tblStart = 0
 $tblEnd = 2000
 $tsStart = 1325347200000  # 2012-01-01 00:00:00.000
+$tsEnd   = 1325347210000
 ###############################################################
 
 sql connect
+$db = db1
+$stb = stb1
+#subtable: tb0 - tb4999
+#print create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(16))
+
+sleep 20000 # wait other client insert data
 
 loop_lable:
 print ====================== client5 start loop query
 
-$db = db2
-$stb = stb2
-
-print create database if not exists $db replica 2
-sql create database if not exists $db replica 2
 sql use $db   
 
-print ==== client4start create table
-$i = $tblStart
-while $i < $tblEnd
-  $tb = dtb . $i
-  sql create table $tb (ts timestamp, c1 int)
-  $i = $i + 1
-endw 
-
-
-print ==== client4start insert, include multi table data in one insert sql
-$totalRows = 0
-$totalRowsPerTbl = 0
-
-$rowsPerLoop = 100
-$ts = $tsStart
-
-$i = $tblStart
-while $i < $tblEnd
-  $tb0 = dtb . $i
-  $i = $i + 1
-  $tb1 = dtb . $i
-  $i = $i + 1
-  $tb2 = dtb . $i
-  $i = $i + 1
-  $tb3 = dtb . $i
-  $i = $i + 1
-  $tb4 = dtb . $i  
-  $i = $i + 1  
-  
-  $x = 0
-  while $x < $rowsPerLoop
-    sql insert into $tb0 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb1 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb2 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb3 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb4 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) 
-    $x = $x + 20
-    $ts = $ts + 40a
-  endw 
-  
-  $totalRowsPerTbl = $totalRowsPerTbl + $x
-  $x = $x * 5
-  $totalRows = $totalRows + $x  
-endw 
-   
-sql select count(*) from tb10
-if $data00 != $totalRowsPerTbl then
-  print data00 $data00 totalRowsPerTbl $totalRowsPerTbl
-  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  print ************ client4   insert loss: $deltaRows *****
-  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-endi
-
-
-print ====client4 start alter table
-$i = $tblStart
-while $i < $tblEnd
-  $tb = dtb . $i
-  sql alter table $tb add c2 float
-  $i = $i + 1
-endw 
-
-print ====client4 continue insert, include multi table data in one insert sql
-
-$i = $tblStart
-while $i < $tblEnd
-  $tb0 = dtb . $i
-  $i = $i + 1
-  $tb1 = dtb . $i
-  $i = $i + 1
-  $tb2 = dtb . $i
-  $i = $i + 1
-  $tb3 = dtb . $i
-  $i = $i + 1
-  $tb4 = dtb . $i  
-  $i = $i + 1  
-  
-  $x = 0
-  while $x < $rowsPerLoop
-    sql insert into $tb0 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb1 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb2 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb3 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb4 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) 
-    $x = $x + 20
-    $ts = $ts + 40a
-  endw 
-  
-  $totalRowsPerTbl = $totalRowsPerTbl + $x
-  $x = $x * 5
-  $totalRows = $totalRows + $x  
-endw 
-   
-sql select count(*) from tb10
-if $data00 != $totalRowsPerTbl then
-  print data00 $data00 totalRowsPerTbl $totalRowsPerTbl
-  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  print ************ client4   insert loss: $deltaRows *****
-  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-endi
+sql select count(*) from $stb
+
+$tsQueryStart = $tsStart
+$tsQueryStart = $tsStart + 90000
+sql select * from $stb where ts > $tsQueryStart and ts < $tsQueryStart order by ts asc limit 500 offset 7
+sql select * from $stb where ts > $tsQueryStart and ts < $tsQueryStart order by ts desc limit 500 offset 7
+
+sql select * from $stb where ts > $tsQueryStart and ts < $tsQueryStart and t2 == 'client1_0'
+sql select * from $stb where ts > $tsQueryStart and ts < $tsQueryStart and t2 == 'client1_1'
+sql select * from $stb where ts > $tsQueryStart and ts < $tsQueryStart and t2 == 'client1_2'
+sql select * from $stb where ts > $tsQueryStart and ts < $tsQueryStart and t2 == 'client1_3'
+
+sql select * from $stb where ts > $tsQueryStart and ts < $tsQueryStart and t2 == 'client2_0'
+sql select * from $stb where ts > $tsQueryStart and ts < $tsQueryStart and t2 == 'client2_1'
+sql select * from $stb where ts > $tsQueryStart and ts < $tsQueryStart and t2 == 'client2_2'
+sql select * from $stb where ts > $tsQueryStart and ts < $tsQueryStart and t2 == 'client2_3'
+
+sql select min(c1)      from $stb
+sql select max(c1)      from $stb
+sql select first(*)     from $stb
+sql select last(*)      from $stb
+sql select last_row(*)  from $stb
+sql select sum(c1)      from $stb
+sql select avg(c1)      from $stb
+
+sql select min(c1)      from tb1
+sql select max(c1)      from tb10
+sql select first(*)     from tb100
+sql select last(*)      from tb1000
+sql select last_row(*)  from tb20
+sql select sum(c1)      from tb200
+sql select avg(c1)      from tb2000
 
-print ====================== client4 drop database
-sql drop if exists database $db 
 goto loop_lable
\ No newline at end of file

tests/script/unique/cluster/cluster_main1.sim

@@ -80,6 +80,7 @@ run_back unique/cluster/main1_client1_2.sim
 run_back unique/cluster/main1_client1_3.sim
 run_back unique/cluster/client3.sim
 run_back unique/cluster/client4.sim
+run_back unique/cluster/client5.sim
 
 sleep 20000
 

tests/script/unique/cluster/cluster_main2.sim

@@ -84,6 +84,7 @@ run_back unique/cluster/main2_client2_2.sim
 run_back unique/cluster/main2_client2_3.sim
 run_back unique/cluster/client3.sim
 run_back unique/cluster/client4.sim
+run_back unique/cluster/client5.sim
 
 sleep 20000