Merge branch '2.0' into feature/2.0tsdb

src/client/inc/tscUtil.h

@@ -85,7 +85,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList,
                                 STableDataBlocks** dataBlocks);
 
 SVnodeSidList*    tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx);
-STableSidExtInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx);
+STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx);
 
 /**
  *

src/client/inc/tsclient.h

@@ -62,7 +62,7 @@ typedef struct STableMeta {
   int8_t     numOfVpeers;
   int16_t    sversion;
   SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT];
-  int32_t    vgid;     // virtual group id, which current table belongs to
+  int32_t    vgId;     // virtual group id, which current table belongs to
   int32_t    sid;      // the index of one table in a virtual node
   uint64_t   uid;      // unique id of a table
   SSchema    schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
@@ -182,7 +182,7 @@ typedef struct STableDataBlocks {
   char    tableId[TSDB_TABLE_ID_LEN];
   int8_t  tsSource;     // where does the UNIX timestamp come from, server or client
   bool    ordered;      // if current rows are ordered or not
-  int64_t vgid;         // virtual group id
+  int64_t vgId;         // virtual group id
   int64_t prevTS;       // previous timestamp, recorded to decide if the records array is ts ascending
   int32_t numOfTables;  // number of tables in current submit block
 

src/client/src/tscLocal.c

@@ -318,15 +318,16 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) {
     SVnodeSidList *pSidList = (SVnodeSidList *)((char *)pMetricMeta + pMetricMeta->list[i]);
 
     for (int32_t j = 0; j < pSidList->numOfSids; ++j) {
-      STableSidExtInfo *pSidExt = tscGetMeterSidInfo(pSidList, j);
+      STableIdInfo *pSidExt = tscGetMeterSidInfo(pSidList, j);
       
       for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutputCols; ++k) {
         SColIndexEx *pColIndex = &tscSqlExprGet(pQueryInfo, k)->colInfo;
         int16_t      offsetId = pColIndex->colIdx;
 
         assert((pColIndex->flag & TSDB_COL_TAG) != 0);
-
-        char *      val = pSidExt->tags + vOffset[offsetId];
+        assert(0);
+        
+        char *      val = NULL;//pSidExt->tags + vOffset[offsetId];
         TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, k);
 
         memcpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, k) * totalNumOfResults + pField->bytes * rowIdx, val,

src/client/src/tscParseInsert.c

@@ -698,7 +698,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char
   SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)(dataBuf->pData);
   tsSetBlockInfo(pBlocks, pTableMeta, numOfRows);
 
-  dataBuf->vgid = pTableMeta->vgid;
+  dataBuf->vgId = pTableMeta->vgId;
   dataBuf->numOfTables = 1;
 
   /*
@@ -1058,7 +1058,6 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
       goto _error_clean;
     }
 
-    void *fp = pSql->fp;
     ptrdiff_t pos = pSql->asyncTblPos - pSql->sqlstr;
     
     if ((code = tscCheckIfCreateTable(&str, pSql)) != TSDB_CODE_SUCCESS) {
@@ -1068,17 +1067,15 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
        * And during the getMeterMetaCallback function, the sql string will be parsed from the
        * interrupted position.
        */
-      if (fp != NULL) {
-        if (TSDB_CODE_ACTION_IN_PROGRESS == code) {
-          tscTrace("async insert and waiting to get meter meta, then continue parse sql from offset: %" PRId64, pos);
-          return code;
-        }
-        
-        // todo add to return
-        tscError("async insert parse error, code:%d, %s", code, tstrerror(code));
-        pSql->asyncTblPos = NULL;
+      if (TSDB_CODE_ACTION_IN_PROGRESS == code) {
+        tscTrace("async insert and waiting to get meter meta, then continue parse sql from offset: %" PRId64, pos);
+        return code;
       }
       
+      // todo add to return
+      tscError("async insert parse error, code:%d, %s", code, tstrerror(code));
+      pSql->asyncTblPos = NULL;
+      
       goto _error_clean;       // TODO: should _clean or _error_clean to async flow ????
     }
 
@@ -1096,15 +1093,13 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
       goto _error_clean;
     }
     
-    int32_t numOfCols = tscGetNumOfTags(pTableMetaInfo->pTableMeta);
     STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
     
     if (sToken.type == TK_VALUES) {
-      SParsedDataColInfo spd = {.numOfCols = numOfCols};
+      SParsedDataColInfo spd = {.numOfCols = tinfo.numOfColumns};
       
       SSchema *pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
-
-      tscSetAssignedColumnInfo(&spd, pSchema, numOfCols);
+      tscSetAssignedColumnInfo(&spd, pSchema, tinfo.numOfColumns);
 
       if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) {
         goto _error_clean;
@@ -1243,7 +1238,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
 
   // submit to more than one vnode
   if (pCmd->pDataBlocks->nSize > 0) {
-    // merge according to vgid
+    // merge according to vgId
     if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) {
       goto _error_clean;
     }

src/client/src/tscSchemaUtil.c

@@ -165,7 +165,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
   
   pTableMeta->sid = pTableMetaMsg->sid;
   pTableMeta->uid = pTableMetaMsg->uid;
-  pTableMeta->vgid = pTableMetaMsg->vgid;
+  pTableMeta->vgId = pTableMetaMsg->vgId;
   
   pTableMeta->numOfVpeers = pTableMetaMsg->numOfVpeers;
   memcpy(pTableMeta->vpeerDesc, pTableMetaMsg->vpeerDesc, sizeof(SVnodeDesc) * pTableMeta->numOfVpeers);

src/client/src/tscServer.c

@@ -341,11 +341,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
      * the tscShouldFreeAsyncSqlObj will success and tscFreeSqlObj free it immediately.
      */
     bool shouldFree = tscShouldFreeAsyncSqlObj(pSql);
-    if (command == TSDB_SQL_INSERT) {  // handle multi-vnode insertion situation
-      (*pSql->fp)(pSql, taosres, rpcMsg->code);
-    } else {
-      (*pSql->fp)(pSql->param, taosres, rpcMsg->code);
-    }
+    (*pSql->fp)(pSql->param, taosres, rpcMsg->code);
 
     if (shouldFree) {
       // If it is failed, all objects allocated during execution taos_connect_a should be released
@@ -539,22 +535,27 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   char *           pMsg, *pStart;
 
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
+  STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta;
   
   pStart = pSql->cmd.payload + tsRpcHeadSize;
   pMsg = pStart;
 
   pShellMsg = (SShellSubmitMsg *)pMsg;
-
+  
+  pShellMsg->desc.numOfVnodes = htonl(1);
+  
   pShellMsg->import = htons(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT) ? 0 : 1);
-  pShellMsg->vnode = 0; //htons(pTableMeta->vpeerDesc[pTableMeta->index].vnode);
-  pShellMsg->numOfSid = htonl(pSql->cmd.numOfTablesInSubmit);  // number of meters to be inserted
+  pShellMsg->header.vgId = htonl(pTableMeta->vgId);
+  pShellMsg->header.contLen = htonl(pSql->cmd.payloadLen);
+  
+  pShellMsg->numOfTables = htonl(pSql->cmd.numOfTablesInSubmit);  // number of meters to be inserted
 
   // pSql->cmd.payloadLen is set during parse sql routine, so we do not use it here
   pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT;
 //  tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pTableMeta->index].ip),
 //           htons(pShellMsg->vnode));
 
-  pSql->cmd.payloadLen = sizeof(SShellSubmitMsg);
+//  pSql->cmd.payloadLen = sizeof(SShellSubmitMsg);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -598,7 +599,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
   SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta;
   SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex);
 
-  int32_t meterInfoSize = (pMetricMeta->tagLen + sizeof(STableSidExtInfo)) * pVnodeSidList->numOfSids;
+  int32_t meterInfoSize = (pMetricMeta->tagLen + sizeof(STableIdInfo)) * pVnodeSidList->numOfSids;
   int32_t outputColumnSize = pQueryInfo->exprsInfo.numOfExprs * sizeof(SSqlFuncExprMsg);
 
   int32_t size = meterInfoSize + outputColumnSize + srcColListSize + exprSize + MIN_QUERY_MSG_PKT_SIZE;
@@ -609,37 +610,34 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
   return size;
 }
 
-static char *doSerializeTableInfo(SSqlObj *pSql, int32_t numOfTables, int32_t vnodeId, char *pMsg) {
+static char *doSerializeTableInfo(SSqlObj *pSql, int32_t numOfTables, int32_t vgId, char *pMsg) {
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0);
 
   STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
   SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta;
 
-  tscTrace("%p vid:%d, query on %d meters", pSql, vnodeId, numOfTables);
+  tscTrace("%p vgId:%d, query on %d tables", pSql, vgId, numOfTables);
   if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
 #ifdef _DEBUG_VIEW
     tscTrace("%p sid:%d, uid:%" PRIu64, pSql, pTableMetaInfo->pTableMeta->sid, pTableMetaInfo->pTableMeta->uid);
 #endif
-    STableSidExtInfo *pTableMetaInfo = (STableSidExtInfo *)pMsg;
-    pTableMetaInfo->sid = htonl(pTableMeta->sid);
-    pTableMetaInfo->uid = htobe64(pTableMeta->uid);
-    pTableMetaInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid));
-    pMsg += sizeof(STableSidExtInfo);
+    STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
+    pTableIdInfo->sid = htonl(pTableMeta->sid);
+    pTableIdInfo->uid = htobe64(pTableMeta->uid);
+    pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid));
+    pMsg += sizeof(STableIdInfo);
   } else {
     SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex);
 
     for (int32_t i = 0; i < numOfTables; ++i) {
-      STableSidExtInfo *pTableMetaInfo = (STableSidExtInfo *)pMsg;
-      STableSidExtInfo *pQueryMeterInfo = tscGetMeterSidInfo(pVnodeSidList, i);
+      STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
+      STableIdInfo *pQueryMeterInfo = tscGetMeterSidInfo(pVnodeSidList, i);
 
-      pTableMetaInfo->sid = htonl(pQueryMeterInfo->sid);
-      pTableMetaInfo->uid = htobe64(pQueryMeterInfo->uid);
-      pTableMetaInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pQueryMeterInfo->uid));
+      pTableIdInfo->sid = htonl(pQueryMeterInfo->sid);
+      pTableIdInfo->uid = htobe64(pQueryMeterInfo->uid);
+      pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pQueryMeterInfo->uid));
       
-      pMsg += sizeof(STableSidExtInfo);
-
-      memcpy(pMsg, pQueryMeterInfo->tags, pMetricMeta->tagLen);
-      pMsg += pMetricMeta->tagLen;
+      pMsg += sizeof(STableIdInfo);
 
 #ifdef _DEBUG_VIEW
       tscTrace("%p sid:%d, uid:%" PRId64, pSql, pQueryMeterInfo->sid, pQueryMeterInfo->uid);
@@ -679,6 +677,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     pQueryMsg->uid = pTableMeta->uid;
     pQueryMsg->numOfTagsCols = 0;
     
+    pQueryMsg->vgId = htonl(pTableMeta->vgId);
     tscTrace("%p queried tables:%d, table id: %s", pSql, 1, pTableMetaInfo->name);
   } else {  // query on super table
     if (pTableMetaInfo->vnodeIndex < 0) {
@@ -696,7 +695,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     }
 
     tscTrace("%p query on vid:%d, number of tables:%d", pSql, vnodeId, numOfTables);
-    pQueryMsg->vnode = htons(vnodeId);
+    pQueryMsg->vgId = htons(vnodeId);
   }
 
   pQueryMsg->numOfTables = htonl(numOfTables);
@@ -764,14 +763,14 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     SColumnBase *pCol = tscColumnBaseInfoGet(&pQueryInfo->colList, i);
     SSchema *    pColSchema = &pSchema[pCol->colIndex.columnIndex];
 
-    if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL ||
-        pColSchema->type > TSDB_DATA_TYPE_NCHAR) {
-      tscError("%p vid:%d sid:%d id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", pSql,
-               htons(pQueryMsg->vnode), pTableMeta->sid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex,
-               pColSchema->name);
-
-      return -1;  // 0 means build msg failed
-    }
+//    if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL ||
+//        pColSchema->type > TSDB_DATA_TYPE_NCHAR) {
+//      tscError("%p vid:%d sid:%d id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", pSql,
+//               htons(pQueryMsg->vnode), pTableMeta->sid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex,
+//               pColSchema->name);
+//
+//      return -1;  // 0 means build msg failed
+//    }
 
     pQueryMsg->colList[i].colId = htons(pColSchema->colId);
     pQueryMsg->colList[i].bytes = htons(pColSchema->bytes);
@@ -865,7 +864,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pQueryMsg->colNameLen = htonl(len);
 
   // serialize the table info (sid, uid, tags)
-  pMsg = doSerializeTableInfo(pSql, numOfTables, htons(pQueryMsg->vnode), pMsg);
+  pMsg = doSerializeTableInfo(pSql, numOfTables, htons(pQueryMsg->vgId), pMsg);
 
   // only include the required tag column schema. If a tag is not required, it won't be sent to vnode
   if (pTableMetaInfo->numOfTags > 0) {
@@ -946,7 +945,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   tscTrace("%p msg built success,len:%d bytes", pSql, msgLen);
   pCmd->payloadLen = msgLen;
   pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY;
-
+  
+  pQueryMsg->contLen = htonl(msgLen);
+  
   assert(msgLen + minMsgSize() <= size);
 
   return TSDB_CODE_SUCCESS;
@@ -1849,12 +1850,12 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
 
   pMetaMsg->sid = htonl(pMetaMsg->sid);
   pMetaMsg->sversion = htons(pMetaMsg->sversion);
-  pMetaMsg->vgid = htonl(pMetaMsg->vgid);
+  pMetaMsg->vgId = htonl(pMetaMsg->vgId);
   pMetaMsg->uid = htobe64(pMetaMsg->uid);
   pMetaMsg->contLen = htons(pMetaMsg->contLen);
 
-  if (pMetaMsg->sid < 0 || pMetaMsg->vgid < 0) {
-    tscError("invalid meter vgid:%d, sid%d", pMetaMsg->vgid, pMetaMsg->sid);
+  if (pMetaMsg->sid < 0 || pMetaMsg->vgId < 0) {
+    tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgId, pMetaMsg->sid);
     return TSDB_CODE_INVALID_VALUE;
   }
 
@@ -1948,11 +1949,11 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
 
     pMeta->sid = htonl(pMeta->sid);
     pMeta->sversion = htons(pMeta->sversion);
-    pMeta->vgid = htonl(pMeta->vgid);
+    pMeta->vgId = htonl(pMeta->vgId);
     pMeta->uid = htobe64(pMeta->uid);
 
-    if (pMeta->sid <= 0 || pMeta->vgid < 0) {
-      tscError("invalid meter vgid:%d, sid%d", pMeta->vgid, pMeta->sid);
+    if (pMeta->sid <= 0 || pMeta->vgId < 0) {
+      tscError("invalid meter vgId:%d, sid%d", pMeta->vgId, pMeta->sid);
       pSql->res.code = TSDB_CODE_INVALID_VALUE;
       pSql->res.numOfTotal = i;
       return TSDB_CODE_OTHERS;
@@ -2067,7 +2068,7 @@ int tscProcessMetricMetaRsp(SSqlObj *pSql) {
     pMeta->numOfVnodes = htonl(pMeta->numOfVnodes);
     pMeta->tagLen = htons(pMeta->tagLen);
 
-    size += pMeta->numOfVnodes * sizeof(SVnodeSidList *) + pMeta->numOfTables * sizeof(STableSidExtInfo *);
+    size += pMeta->numOfVnodes * sizeof(SVnodeSidList *) + pMeta->numOfTables * sizeof(STableIdInfo *);
 
     char *pBuf = calloc(1, size);
     if (pBuf == NULL) {
@@ -2093,16 +2094,16 @@ int tscProcessMetricMetaRsp(SSqlObj *pSql) {
 
       tscTrace("%p metricmeta:vid:%d,numOfTables:%d", pSql, i, pLists->numOfSids);
 
-      pBuf += sizeof(SVnodeSidList) + sizeof(STableSidExtInfo *) * pSidLists->numOfSids;
+      pBuf += sizeof(SVnodeSidList) + sizeof(STableIdInfo *) * pSidLists->numOfSids;
       rsp += sizeof(SVnodeSidList);
 
-      size_t elemSize = sizeof(STableSidExtInfo) + pNewMetricMeta->tagLen;
+      size_t elemSize = sizeof(STableIdInfo) + pNewMetricMeta->tagLen;
       for (int32_t j = 0; j < pSidLists->numOfSids; ++j) {
         pLists->pSidExtInfoList[j] = pBuf - (char *)pLists;
         memcpy(pBuf, rsp, elemSize);
 
-        ((STableSidExtInfo *)pBuf)->uid = htobe64(((STableSidExtInfo *)pBuf)->uid);
-        ((STableSidExtInfo *)pBuf)->sid = htonl(((STableSidExtInfo *)pBuf)->sid);
+        ((STableIdInfo *)pBuf)->uid = htobe64(((STableIdInfo *)pBuf)->uid);
+        ((STableIdInfo *)pBuf)->sid = htonl(((STableIdInfo *)pBuf)->sid);
 
         rsp += elemSize;
         pBuf += elemSize;

src/client/src/tscSql.c

@@ -130,7 +130,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
   pSql->pTscObj = pObj;
   pSql->signature = pSql;
   tsem_init(&pSql->rspSem, 0, 0);
-//  tsem_init(&pSql->emptyRspSem, 0, 1);
+  
   pObj->pSql = pSql;
   pSql->fp = fp;
   pSql->param = param;
@@ -146,6 +146,8 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
     return NULL;
   }
 
+  // tsRpcHeaderSize will be updated during RPC initialization, so only after it initialization, this value is valid
+  tsInsertHeadSize = tsRpcHeadSize + sizeof(SShellSubmitMsg);
   return pObj;
 }
 

src/client/src/tscSub.c

@@ -202,7 +202,7 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
     for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) {
       SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i);
       for (int32_t j = 0; j < pVnodeSidList->numOfSids; j++) {
-        STableSidExtInfo *pTableMetaInfo = tscGetMeterSidInfo(pVnodeSidList, j);
+        STableIdInfo *pTableMetaInfo = tscGetMeterSidInfo(pVnodeSidList, j);
         int64_t uid = pTableMetaInfo->uid;
         progress[numOfTables].uid = uid;
         progress[numOfTables++].key = tscGetSubscriptionProgress(pSub, uid);

src/client/src/tscSystem.c

@@ -34,7 +34,6 @@ void *  pTscMgmtConn;
 void *  pSlaveConn;
 void *  tscCacheHandle;
 int32_t globalCode = 0;
-int     initialized = 0;
 int     slaveIndex;
 void *  tscTmr;
 void *  tscQhandle;
@@ -187,9 +186,7 @@ void taos_init_imp() {
 
   if (tscCacheHandle == NULL) tscCacheHandle = taosCacheInit(tscTmr, refreshTime);
 
-  initialized = 1;
   tscTrace("client is initialized successfully");
-  tsInsertHeadSize = tsRpcHeadSize + sizeof(SShellSubmitMsg);
 }
 
 void taos_init() { pthread_once(&tscinit, taos_init_imp); }

src/client/src/tscUtil.c

@@ -191,7 +191,7 @@ SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx
   return (SVnodeSidList*)(pMetricmeta->list[vnodeIdx] + (char*)pMetricmeta);
 }
 
-STableSidExtInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) {
+STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) {
   if (pSidList == NULL) {
     tscError("illegal sidlist");
     return 0;
@@ -206,7 +206,7 @@ STableSidExtInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) {
   
   assert(pSidList->pSidExtInfoList[idx] >= 0);
   
-  return (STableSidExtInfo*)(pSidList->pSidExtInfoList[idx] + (char*)pSidList);
+  return (STableIdInfo*)(pSidList->pSidExtInfoList[idx] + (char*)pSidList);
 }
 
 bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) {
@@ -614,7 +614,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
    */
   pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize;
 
-  assert(pCmd->allocSize >= pCmd->payloadLen + tsRpcHeadSize + 100);
+  assert(pCmd->allocSize >= pCmd->payloadLen + tsRpcHeadSize + 100 && pCmd->payloadLen > 0);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -705,8 +705,9 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
     STableDataBlocks* pOneTableBlock = pTableDataBlockList->pData[i];
 
     STableDataBlocks* dataBuf = NULL;
-    int32_t           ret =
-        tscGetDataBlockFromList(pVnodeDataBlockHashList, pVnodeDataBlockList, pOneTableBlock->vgid, TSDB_PAYLOAD_SIZE,
+    
+    int32_t ret =
+        tscGetDataBlockFromList(pVnodeDataBlockHashList, pVnodeDataBlockList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE,
                                 tsInsertHeadSize, 0, pOneTableBlock->tableId, pOneTableBlock->pTableMeta, &dataBuf);
     if (ret != TSDB_CODE_SUCCESS) {
       tscError("%p failed to prepare the data block buffer for merging table data, code:%d", pSql, ret);

src/dnode/src/dnodeRead.c

@@ -15,13 +15,16 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
+
 #include "taoserror.h"
 #include "taosmsg.h"
 #include "tlog.h"
 #include "tqueue.h"
 #include "trpc.h"
-#include "dnodeRead.h"
+
 #include "dnodeMgmt.h"
+#include "dnodeRead.h"
+#include "queryExecutor.h"
 
 typedef struct {
   int32_t  code;
@@ -88,7 +91,7 @@ void dnodeRead(SRpcMsg *pMsg) {
 
   while (leftLen > 0) {
     SMsgHead *pHead = (SMsgHead *) pCont;
-    pHead->vgId    = 1; //htonl(pHead->vgId);
+    pHead->vgId    = 1;//htonl(pHead->vgId);
     pHead->contLen = pMsg->contLen; //htonl(pHead->contLen);
 
     void *pVnode = dnodeGetVnode(pHead->vgId);
@@ -223,20 +226,26 @@ static void dnodeProcessReadResult(SReadMsg *pRead) {
 }
 
 static void dnodeProcessQueryMsg(SReadMsg *pMsg) {
-  void *pQInfo = (void*)100;
-  dTrace("query msg is disposed, qInfo:%p", pQInfo);
-
+  SQueryTableMsg* pQueryTableMsg = (SQueryTableMsg*) pMsg->pCont;
+  
+  SQInfo* pQInfo = NULL;
+  int32_t code = qCreateQueryInfo(pQueryTableMsg, &pQInfo);
+  
   SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp));
-  pRsp->code    = 0;
+  pRsp->code    = code;
   pRsp->qhandle = htobe64((uint64_t) (pQInfo));
 
   SRpcMsg rpcRsp = {
       .handle = pMsg->rpcMsg.handle,
       .pCont = pRsp,
       .contLen = sizeof(SQueryTableRsp),
-      .code = 0,
+      .code = code,
       .msgType = 0
   };
+  
+  // do execute query
+  qTableQuery(pQInfo);
+  
   rpcSendResponse(&rpcRsp);
 }
 
@@ -245,21 +254,51 @@ static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) {
   void *pQInfo = htobe64(pRetrieve->qhandle);
 
   dTrace("retrieve msg is disposed, qInfo:%p", pQInfo);
-
-  assert(pQInfo != NULL);
-  int32_t contLen = 100;
-  SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *) rpcMallocCont(contLen);
-  pRsp->numOfRows = 0;
-  pRsp->precision = 0;
-  pRsp->offset    = 0;
-  pRsp->useconds  = 0;
-
-  SRpcMsg rpcRsp = {
-      .handle = pMsg->rpcMsg.handle,
-      .pCont = pRsp,
-      .contLen = contLen,
-      .code = 0,
-      .msgType = 0
-  };
+  
+  int32_t rowSize = 0;
+  int32_t numOfRows = 0;
+  int32_t contLen = 0;
+  
+  SRpcMsg rpcRsp = {0};
+  
+  int32_t code = qRetrieveQueryResultInfo(pQInfo, &numOfRows, &rowSize);
+  if (code != TSDB_CODE_SUCCESS) {
+    contLen = sizeof(SRetrieveTableRsp);
+    
+    SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen);
+    pRsp->numOfRows = 0;
+    pRsp->precision = 0;
+    pRsp->offset = 0;
+    pRsp->useconds = 0;
+  
+    rpcRsp = (SRpcMsg) {
+        .handle = pMsg->rpcMsg.handle,
+        .pCont = pRsp,
+        .contLen = contLen,
+        .code = code,
+        .msgType = 0
+    };
+    
+    //todo free qinfo
+  } else {
+    contLen = 100;
+    
+    SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen);
+    pRsp->numOfRows = 0;
+    pRsp->precision = 0;
+    pRsp->offset = 0;
+    pRsp->useconds = 0;
+
+    *(int64_t*) pRsp->data = 1000;
+    
+    rpcRsp = (SRpcMsg) {
+        .handle = pMsg->rpcMsg.handle,
+        .pCont = pRsp,
+        .contLen = contLen,
+        .code = code,
+        .msgType = 0
+    };
+  }
+  
   rpcSendResponse(&rpcRsp);
 }

src/dnode/src/dnodeWrite.c

@@ -275,7 +275,9 @@ static void dnodeProcessSubmitMsg(SWriteMsg *pMsg) {
   pRsp->numOfRows         = htonl(1);
   pRsp->affectedRows      = htonl(1);
   pRsp->numOfFailedBlocks = 0;
-
+  
+  // todo write to tsdb
+  
   SRpcMsg rpcRsp = {
     .handle = pMsg->rpcMsg.handle,
     .pCont = pRsp,

src/inc/taosmsg.h

@@ -198,10 +198,20 @@ typedef struct {
 } SShellSubmitBlock;
 
 typedef struct {
+  int32_t numOfVnodes;
+} SMsgDesc;
+
+typedef struct SMsgHead {
+  int32_t contLen;
+  int32_t vgId;
+} SMsgHead;
+
+typedef struct {
+  SMsgDesc desc;
+  SMsgHead header;
   int16_t import;
-  int16_t vnode;
-  int32_t numOfSid; /* total number of sid */
-  char    blks[];   /* numOfSid blocks, each blocks for one table */
+  int32_t numOfTables; // total number of sid
+  char    blks[];      // number of data blocks, each table has at least one data block
 } SShellSubmitMsg;
 
 typedef struct {
@@ -232,15 +242,6 @@ typedef struct {
   uint32_t ip;
 } SVnodeDesc;
 
-typedef struct {
-  int32_t numOfVnodes;
-} SMsgDesc;
-
-typedef struct {
-  int32_t contLen;
-  int32_t vgId;
-} SMsgHead;
-
 typedef struct {
   int32_t    contLen;
   int32_t    vgId;
@@ -434,15 +435,11 @@ typedef struct SColumnInfo {
   SColumnFilterInfo *filters;
 } SColumnInfo;
 
-/*
- * enable vnode to understand how to group several tables with different tag;
- */
-typedef struct STableSidExtInfo {
+typedef struct STableIdInfo {
   int32_t sid;
   int64_t uid;
-  TSKEY   key;   // key for subscription
-  char    tags[];
-} STableSidExtInfo;
+  TSKEY   key;   // last accessed ts, for subscription
+} STableIdInfo;
 
 typedef struct STimeWindow {
   TSKEY skey;
@@ -455,10 +452,10 @@ typedef struct STimeWindow {
  * the outputCols will be 3 while the numOfCols is 1.
  */
 typedef struct {
-  int16_t  vnode;
+  int32_t contLen;   // msg header
+  int16_t vgId;
+  
   int32_t  numOfTables;
-  uint64_t pSidExtInfo;  // table id & tag info ptr, in windows pointer may
-
   uint64_t uid;
   STimeWindow window;
 
@@ -504,7 +501,7 @@ typedef struct {
 } SQueryTableMsg;
 
 typedef struct {
-  char     code;
+  int32_t  code;
   uint64_t qhandle;
 } SQueryTableRsp;
 
@@ -670,7 +667,7 @@ typedef struct {
   SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT];
   int16_t    index;  // used locally
   int32_t    numOfSids;
-  int32_t    pSidExtInfoList[];  // offset value of STableSidExtInfo
+  int32_t    pSidExtInfoList[];  // offset value of STableIdInfo
 } SVnodeSidList;
 
 typedef struct {
@@ -692,7 +689,7 @@ typedef struct STableMetaMsg {
   int8_t  numOfVpeers;
   SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT];
   int32_t  sid;
-  int32_t  vgid;
+  int32_t  vgId;
   uint64_t uid;
   SSchema  schema[];
 } STableMetaMsg;

src/mnode/src/mgmtChildTable.c

@@ -445,7 +445,7 @@ int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName
 int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) {
   pMeta->uid          = htobe64(pTable->uid);
   pMeta->sid          = htonl(pTable->sid);
-  pMeta->vgid         = htonl(pTable->vgId);
+  pMeta->vgId         = htonl(pTable->vgId);
   pMeta->sversion     = htons(pTable->superTable->sversion);
   pMeta->precision    = pDb->cfg.precision;
   pMeta->numOfTags    = pTable->superTable->numOfTags;

src/mnode/src/mgmtDnode.c

@@ -553,11 +553,14 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
   pDnode->privateIp        = htonl(pStatus->privateIp);
   pDnode->publicIp         = htonl(pStatus->publicIp);
   pDnode->lastReboot       = htonl(pStatus->lastReboot);
-  pDnode->numOfTotalVnodes = htons(pStatus->numOfTotalVnodes);
   pDnode->numOfCores       = htons(pStatus->numOfCores);
   pDnode->diskAvailable    = pStatus->diskAvailable;
   pDnode->alternativeRole  = pStatus->alternativeRole;
 
+  if (pDnode->numOfTotalVnodes == 0) {
+    pDnode->numOfTotalVnodes = htons(pStatus->numOfTotalVnodes); 
+  }
+  
   if (pStatus->dnodeId == 0) {
     mTrace("dnode:%d, first access, privateIp:%s, name:%s, ", pDnode->dnodeId, taosIpStr(pDnode->privateIp), pDnode->dnodeName);
     mgmtSetDnodeMaxVnodes(pDnode);

src/mnode/src/mgmtNormalTable.c

@@ -524,7 +524,7 @@ static int32_t mgmtSetSchemaFromNormalTable(SSchema *pSchema, SNormalTableObj *p
 int32_t mgmtGetNormalTableMeta(SDbObj *pDb, SNormalTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) {
   pMeta->uid          = htobe64(pTable->uid);
   pMeta->sid          = htonl(pTable->sid);
-  pMeta->vgid         = htonl(pTable->vgId);
+  pMeta->vgId         = htonl(pTable->vgId);
   pMeta->sversion     = htons(pTable->sversion);
   pMeta->precision    = pDb->cfg.precision;
   pMeta->numOfTags    = 0;

src/mnode/src/mgmtSuperTable.c

@@ -654,7 +654,7 @@ int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) {
 int32_t mgmtGetSuperTableMeta(SDbObj *pDb, SSuperTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) {
   pMeta->uid          = htobe64(pTable->uid);
   pMeta->sid          = htonl(pTable->sid);
-  pMeta->vgid         = htonl(pTable->vgId);
+  pMeta->vgId         = htonl(pTable->vgId);
   pMeta->sversion     = htons(pTable->sversion);
   pMeta->precision    = pDb->cfg.precision;
   pMeta->numOfTags    = pTable->numOfTags;

src/query/CMakeLists.txt

@@ -11,5 +11,5 @@ INCLUDE_DIRECTORIES(inc)
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(query ${SRC})
-  TARGET_LINK_LIBRARIES(query tutil m rt)
+  TARGET_LINK_LIBRARIES(query tsdb tutil m rt)
 ENDIF ()
\ No newline at end of file

src/query/inc/qextbuffer.h

@@ -124,9 +124,8 @@ typedef struct tTagSchema {
 typedef struct tSidSet {
   int32_t            numOfSids;
   int32_t            numOfSubSet;
-  STableSidExtInfo **pSids;
+  STableIdInfo     **pTableIdList;
   int32_t *          starterPos;  // position of each subgroup, generated according to
-
   SColumnModel      *pColumnModel;
   SColumnOrderInfo   orderIdx;
 } tSidSet;

src/query/inc/queryExecutor.h

@@ -25,6 +25,7 @@
 #include "taosdef.h"
 #include "tref.h"
 #include "tsqlfunction.h"
+#include "tarray.h"
 
 typedef struct SData {
   int32_t num;
@@ -39,7 +40,7 @@ enum {
 
 struct SColumnFilterElem;
 typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2);
-typedef int (*__block_search_fn_t)(char* data, int num, int64_t key, int order);
+typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
 
 typedef struct SSqlGroupbyExpr {
   int16_t     tableIndex;
@@ -142,7 +143,7 @@ typedef struct SQuery {
   int32_t           pos;
   int64_t           pointsOffset;  // the number of points offset to save read data
   SData**           sdata;
-
+  int32_t capacity;
   SSingleColumnFilterInfo* pFilterInfo;
 } SQuery;
 
@@ -152,7 +153,6 @@ typedef struct SQueryCostSummary {
 typedef struct SQueryRuntimeEnv {
   SResultInfo*       resultInfo;  // todo refactor to merge with SWindowResInfo
   SQuery*            pQuery;
-  void*              pTabObj;
   SData**            pInterpoBuf;
   SQLFunctionCtx*    pCtx;
   int16_t            numOfRowsPerPage;
@@ -174,16 +174,17 @@ typedef struct SQInfo {
   TSKEY            startTime;
   int64_t          elapsedTime;
   SResultRec       rec;
-  int              pointsReturned;
-  int              pointsInterpo;
-  int              code;  // error code to returned to client
+  int32_t          pointsReturned;
+  int32_t          pointsInterpo;
+  int32_t          code;   // error code to returned to client
+  int32_t          killed; // denotes if current query is killed
   sem_t            dataReady;
-  SHashObj*        pTableList;  // table list
+  SArray*          pTableIdList;  // table list
   SQueryRuntimeEnv runtimeEnv;
   int32_t          subgroupIdx;
   int32_t          offset; /* offset in group result set of subgroup */
-  tSidSet*         pSidSet;
-
+//  tSidSet*         pSidSet;
+  
   T_REF_DECLARE()
   /*
    * the query is executed position on which meter of the whole list.
@@ -204,13 +205,13 @@ typedef struct SQInfo {
  * @param pQInfo
  * @return
  */
-int32_t qCreateQueryInfo(void* pReadMsg, SQInfo** pQInfo);
+int32_t qCreateQueryInfo(SQueryTableMsg* pQueryTableMsg, SQInfo** pQInfo);
 
 /**
  * query on single table
  * @param pReadMsg
  */
-void qTableQuery(void* pReadMsg);
+void qTableQuery(SQInfo* pQInfo);
 
 /**
  * query on super table
@@ -218,4 +219,13 @@ void qTableQuery(void* pReadMsg);
  */
 void qSuperTableQuery(void* pReadMsg);
 
+/**
+ * wait for the query completed, and retrieve final results to client
+ * @param pQInfo
+ */
+int32_t qRetrieveQueryResultInfo(SQInfo* pQInfo, int32_t *numOfRows, int32_t* rowsize);
+
+
+//int32_t qBuildQueryResult(SQInfo* pQInfo, void* pBuf);
+
 #endif  // TDENGINE_QUERYEXECUTOR_H

src/rpc/src/rpcMain.c

@@ -287,7 +287,7 @@ void rpcClose(void *param) {
   (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle);
 
   for (int i = 0; i < pRpc->sessions; ++i) {
-    if (pRpc->connList[i].user[0]) {
+    if (pRpc->connList && pRpc->connList[i].user[0]) {
       rpcCloseConn((void *)(pRpc->connList + i));
     }
   }
@@ -495,35 +495,35 @@ static void rpcCloseConn(void *thandle) {
   SRpcConn *pConn = (SRpcConn *)thandle;
   SRpcInfo *pRpc = pConn->pRpc;
 
-  rpcLockConn(pConn);
+  if (pConn->user[0] == 0) return;
 
-  if (pConn->user[0]) {
-    pConn->user[0] = 0;
-    if (taosCloseConn[pConn->connType]) (*taosCloseConn[pConn->connType])(pConn->chandle);
-
-    taosTmrStopA(&pConn->pTimer);
-    taosTmrStopA(&pConn->pIdleTimer);
-
-    if ( pRpc->connType == TAOS_CONN_SERVER) {
-      char hashstr[40] = {0};
-      sprintf(hashstr, "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType);
-      taosDeleteStrHash(pRpc->hash, hashstr);
-      rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg
-      pConn->pRspMsg = NULL;
-      pConn->inType = 0;
-      pConn->inTranId = 0;
-    } else {
-      pConn->outType = 0;
-      pConn->outTranId = 0;
-      pConn->pReqMsg = NULL;
-    }
+  rpcLockConn(pConn);
 
-    taosFreeId(pRpc->idPool, pConn->sid);
-    pConn->pContext = NULL;
+  pConn->user[0] = 0;
+  if (taosCloseConn[pConn->connType]) (*taosCloseConn[pConn->connType])(pConn->chandle);
 
-    tTrace("%s %p, rpc connection is closed", pRpc->label, pConn);
+  taosTmrStopA(&pConn->pTimer);
+  taosTmrStopA(&pConn->pIdleTimer);
+
+  if ( pRpc->connType == TAOS_CONN_SERVER) {
+    char hashstr[40] = {0};
+    sprintf(hashstr, "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType);
+    taosDeleteStrHash(pRpc->hash, hashstr);
+    rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg
+    pConn->pRspMsg = NULL;
+    pConn->inType = 0;
+    pConn->inTranId = 0;
+  } else {
+    pConn->outType = 0;
+    pConn->outTranId = 0;
+    pConn->pReqMsg = NULL;
   }
 
+  taosFreeId(pRpc->idPool, pConn->sid);
+  pConn->pContext = NULL;
+
+  tTrace("%s %p, rpc connection is closed", pRpc->label, pConn);
+
   rpcUnlockConn(pConn);
 }
 

src/vnode/tsdb/src/tsdbRead.c

@@ -16,3 +16,51 @@
 #include "os.h"
 #include "tsdb.h"
 
+tsdb_query_handle_t *tsdbQueryByTableId(STsdbQueryCond *pCond, SArray *idList, SArray *pColumnInfo) {
+
+}
+
+bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) {
+  return false;
+}
+
+SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) {
+
+}
+
+int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SDataStatis **pBlockStatis) {
+
+}
+
+SArray *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle, SArray *pIdList) {
+
+}
+
+int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, STimeWindow* window, tsdbpos_t position, int16_t order) {
+
+}
+
+int32_t tsdbDataBlockSeek(tsdb_query_handle_t *pQueryHandle, tsdbpos_t pos) {
+
+}
+
+tsdbpos_t tsdbDataBlockTell(tsdb_query_handle_t *pQueryHandle) {
+  return NULL;
+}
+
+SArray *tsdbRetrieveDataRow(tsdb_query_handle_t *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond) {
+
+}
+
+tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr) {
+
+}
+
+STableIDList *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle) {
+
+}
+
+STableIDList *tsdbQueryTableList(int16_t stableId, const char *pTagCond) {
+
+}
+