Merge pull request #10546 from taosdata/feature/3.0_liaohj

Feature/3.0 liaohj

include/common/tcommon.h

@@ -25,24 +25,6 @@
 extern "C" {
 #endif
 
-// typedef struct STimeWindow {
-//   TSKEY skey;
-//   TSKEY ekey;
-// } STimeWindow;
-
-// typedef struct {
-//   int32_t dataLen;
-//   char    name[TSDB_TABLE_FNAME_LEN];
-//   char   *data;
-// } STagData;
-
-// typedef struct SSchema {
-//   uint8_t type;
-//   char    name[TSDB_COL_NAME_LEN];
-//   int16_t colId;
-//   int16_t bytes;
-// } SSchema;
-
 enum {
   TMQ_CONF__RESET_OFFSET__LATEST = -1,
   TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,

include/libs/executor/executor.h

@@ -89,25 +89,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds);
  */
 int32_t qRetrieveQueryResultInfo(qTaskInfo_t tinfo, bool* buildRes, void* pRspContext);
 
-/**
- *
- * Retrieve the actual results to fill the response message payload.
- * Note that this function must be executed after qRetrieveQueryResultInfo is invoked.
- *
- * @param tinfo  tinfo object
- * @param pRsp    response message
- * @param contLen payload length
- * @return
- */
-//int32_t qDumpRetrieveResult(qTaskInfo_t tinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec);
-
-/**
- * return the transporter context (RPC)
- * @param tinfo
- * @return
- */
-void* qGetResultRetrieveMsg(qTaskInfo_t tinfo);
-
 /**
  * kill the ongoing query and free the query handle and corresponding resources automatically
  * @param tinfo  qhandle

include/libs/scalar/filter.h

@@ -19,10 +19,12 @@
 extern "C" {
 #endif
 
+#include "tcommon.h"
+#include "nodes.h"
+
 typedef struct SFilterInfo SFilterInfo;
 typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
 
-
 enum {
   FLT_OPTION_NO_REWRITE = 1,
   FLT_OPTION_TIMESTAMP = 2,
@@ -34,7 +36,6 @@ typedef struct SFilterColumnParam{
   SArray* pDataBlock;
 } SFilterColumnParam;
 
-
 extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options);
 extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
 extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param);

include/libs/sync/sync.h

@@ -21,7 +21,7 @@ extern "C" {
 #endif
 
 #include <stdint.h>
-#include <tep.h>
+#include <tdatablock.h>
 #include "taosdef.h"
 #include "trpc.h"
 

include/util/thash.h

@@ -28,7 +28,8 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
 typedef void (*_hash_before_fn_t)(void *);
 typedef void (*_hash_free_fn_t)(void *);
 
-#define HASH_NODE_EXIST(code) (code == -2)
+#define HASH_KEY_ALREADY_EXISTS (-2)
+#define HASH_NODE_EXIST(code) (code == HASH_KEY_ALREADY_EXISTS)
 
 /**
  * murmur hash algorithm
@@ -49,24 +50,14 @@ uint32_t taosIntHash_32(const char *key, uint32_t len);
 uint32_t taosIntHash_64(const char *key, uint32_t len);
 
 _hash_fn_t taosGetDefaultHashFunction(int32_t type);
-
 _equal_fn_t taosGetDefaultEqualFunction(int32_t type);
 
-typedef struct SHashNode {
-  struct SHashNode *next;
-  uint32_t          hashVal;  // the hash value of key
-  uint32_t          dataLen;  // length of data
-  uint32_t          keyLen;   // length of the key
-  uint16_t          refCount; // reference count
-  int8_t            removed;  // flag to indicate removed
-  char              data[];
-} SHashNode;
-
 typedef enum SHashLockTypeE {
   HASH_NO_LOCK = 0,
   HASH_ENTRY_LOCK = 1,
 } SHashLockTypeE;
 
+typedef struct SHashNode SHashNode;
 typedef struct SHashObj SHashObj;
 
 /**

source/client/inc/clientInt.h

@@ -20,12 +20,12 @@
 extern "C" {
 #endif
 
-#include "tcommon.h"
 #include "parser.h"
 #include "query.h"
 #include "taos.h"
+#include "tcommon.h"
+#include "tdatablock.h"
 #include "tdef.h"
-#include "tep.h"
 #include "thash.h"
 #include "tlist.h"
 #include "tmsg.h"

source/client/src/clientImpl.c

@@ -4,8 +4,8 @@
 #include "parser.h"
 #include "planner.h"
 #include "scheduler.h"
+#include "tdatablock.h"
 #include "tdef.h"
-#include "tep.h"
 #include "tglobal.h"
 #include "tmsgtype.h"
 #include "tpagedbuf.h"

source/client/src/tmq.c

@@ -20,8 +20,8 @@
 #include "parser.h"
 #include "planner.h"
 #include "scheduler.h"
+#include "tdatablock.h"
 #include "tdef.h"
-#include "tep.h"
 #include "tglobal.h"
 #include "tmsgtype.h"
 #include "tpagedbuf.h"

source/common/src/tep.c → source/common/src/tdatablock.c

@@ -14,7 +14,7 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "tep.h"
+#include "tdatablock.h"
 #include "tcompare.h"
 #include "tglobal.h"
 

source/common/src/tglobal.c

@@ -17,7 +17,7 @@
 #include "tglobal.h"
 #include "tcompare.h"
 #include "tconfig.h"
-#include "tep.h"
+#include "tdatablock.h"
 #include "tlog.h"
 
 SConfig *tsCfg = NULL;

source/common/test/commonTests.cpp

@@ -8,7 +8,8 @@
 #pragma GCC diagnostic ignored "-Wsign-compare"
 #include "os.h"
 
-#include "tep.h"
+#include "tcommon.h"
+#include "tdatablock.h"
 #include "tcommon.h"
 #include "taos.h"
 #include "tvariant.h"

source/dnode/mgmt/impl/inc/dndInt.h

@@ -23,9 +23,11 @@ extern "C" {
 #include "os.h"
 
 #include "cJSON.h"
+#include "monitor.h"
 #include "tcache.h"
 #include "tcrc32c.h"
-#include "tep.h"
+#include "tdatablock.h"
+#include "tglobal.h"
 #include "thash.h"
 #include "tlockfree.h"
 #include "tlog.h"
@@ -35,8 +37,6 @@ extern "C" {
 #include "tthread.h"
 #include "ttime.h"
 #include "tworker.h"
-#include "tglobal.h"
-#include "monitor.h"
 
 #include "dnode.h"
 

source/dnode/mgmt/impl/test/sut/src/client.cpp

@@ -14,7 +14,7 @@
  */
 
 #include "sut.h"
-#include "tep.h"
+#include "tdatablock.h"
 
 static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) {
   TestClient* client = (TestClient*)parent;

source/dnode/mnode/impl/inc/mndInt.h

@@ -20,7 +20,7 @@
 
 #include "sdb.h"
 #include "tcache.h"
-#include "tep.h"
+#include "tdatablock.h"
 #include "tglobal.h"
 #include "tqueue.h"
 #include "ttime.h"

source/libs/catalog/test/catalogTests.cpp

@@ -27,13 +27,13 @@
 #include "os.h"
 #include "tglobal.h"
 #include "catalog.h"
+#include "catalogInt.h"
 #include "stub.h"
 #include "taos.h"
+#include "tdatablock.h"
 #include "tdef.h"
-#include "tep.h"
 #include "trpc.h"
 #include "tvariant.h"
-#include "catalogInt.h"
 
 namespace {
 

source/libs/executor/inc/executorimpl.h

@@ -15,12 +15,12 @@
 #ifndef TDENGINE_EXECUTORIMPL_H
 #define TDENGINE_EXECUTORIMPL_H
 
-#include "tsort.h"
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 #include "os.h"
+#include "tsort.h"
 #include "tcommon.h"
 #include "tlosertree.h"
 #include "ttszip.h"
@@ -157,6 +157,13 @@ typedef struct STaskCostInfo {
   SHashObj* operatorProfResults;  // map<operator_type, SQueryProfEvent>
 } STaskCostInfo;
 
+typedef struct SOperatorCostInfo {
+  uint64_t openCost;
+  uint64_t execCost;
+  uint64_t totalRows;
+  uint64_t totalBytes;
+} SOperatorCostInfo;
+
 typedef struct {
   int64_t vgroupLimit;
   int64_t ts;

source/libs/executor/src/executorMain.c

@@ -179,13 +179,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
   return pTaskInfo->code;
 }
 
-void* qGetResultRetrieveMsg(qTaskInfo_t qinfo) {
-  SQInfo* pQInfo = (SQInfo*) qinfo;
-  assert(pQInfo != NULL);
-
-  return pQInfo->rspContext;
-}
-
 int32_t qKillTask(qTaskInfo_t qinfo) {
   SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo;
 
@@ -221,7 +214,7 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
 int32_t qIsTaskCompleted(qTaskInfo_t qinfo) {
   SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo;
 
-  if (pTaskInfo == NULL /*|| !isValidQInfo(pTaskInfo)*/) {
+  if (pTaskInfo == NULL) {
     return TSDB_CODE_QRY_INVALID_QHANDLE;
   }
 
@@ -235,33 +228,3 @@ void qDestroyTask(qTaskInfo_t qTaskHandle) {
   queryCostStatis(pTaskInfo);   // print the query cost summary
   doDestroyTask(pTaskInfo);
 }
-
-#if 0
-//kill by qid
-int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount) {
-  int32_t error = TSDB_CODE_SUCCESS;
-  void** handle = qAcquireTask(pMgmt, qId);
-  if(handle == NULL) return terrno;
-
-  SQInfo* pQInfo = (SQInfo*)(*handle);
-  if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
-    return TSDB_CODE_QRY_INVALID_QHANDLE;
-  }
-  qWarn("%s be killed(no memory commit).", pQInfo->qId);
-  setTaskKilled(pQInfo);
-
-  // wait query stop
-  int32_t loop = 0;
-  while (pQInfo->owner != 0) {
-    taosMsleep(waitMs);
-    if(loop++ > waitCount){
-      error = TSDB_CODE_FAILED;
-      break;
-    }
-  }
-
-  qReleaseTask(pMgmt, (void **)&handle, true);
-  return error;
-}
-
-#endif

source/libs/executor/src/executorimpl.c

@@ -15,12 +15,12 @@
 
 #include "os.h"
 
-#include "tep.h"
-#include "tsort.h"
-#include "texception.h"
 #include "parser.h"
+#include "tdatablock.h"
+#include "texception.h"
 #include "tglobal.h"
 #include "tmsg.h"
+#include "tsort.h"
 #include "ttime.h"
 
 #include "executorimpl.h"
@@ -8730,10 +8730,8 @@ static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type
 
 static int64_t getQuerySupportBufSize(size_t numOfTables) {
   size_t s1 = sizeof(STableQueryInfo);
-  size_t s2 = sizeof(SHashNode);
-
 //  size_t s3 = sizeof(STableCheckInfo);  buffer consumption in tsdb
-  return (int64_t)((s1 + s2) * 1.5 * numOfTables);
+  return (int64_t)(s1* 1.5 * numOfTables);
 }
 
 int32_t checkForQueryBuf(size_t numOfTables) {

source/libs/executor/src/tsort.c

@@ -16,11 +16,11 @@
 #include "tcommon.h"
 #include "query.h"
 
-#include "tsort.h"
-#include "tep.h"
+#include "tdatablock.h"
 #include "tdef.h"
 #include "tlosertree.h"
 #include "tpagedbuf.h"
+#include "tsort.h"
 #include "tutil.h"
 
 typedef struct STupleHandle {

source/libs/executor/test/executorTests.cpp

@@ -29,7 +29,7 @@
 #include "taos.h"
 #include "tdef.h"
 #include "tvariant.h"
-#include "tep.h"
+#include "tdatablock.h"
 #include "trpc.h"
 #include "stub.h"
 #include "executor.h"

source/libs/executor/test/sortTests.cpp

@@ -29,8 +29,8 @@
 #include "executor.h"
 #include "stub.h"
 #include "taos.h"
+#include "tdatablock.h"
 #include "tdef.h"
-#include "tep.h"
 #include "trpc.h"
 #include "tvariant.h"
 

source/libs/function/src/taggfunction.c

@@ -28,8 +28,8 @@
 #include "tbuffer.h"
 #include "tcompression.h"
 //#include "queryLog.h"
+#include "tdatablock.h"
 #include "tudf.h"
-#include "tep.h"
 
 #define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput))
 #define GET_INPUT_DATA(x, y) ((char*) colDataGetData((x)->pInput, (y)))

source/libs/parser/test/mockCatalogService.cpp

@@ -15,10 +15,10 @@
 
 #include "mockCatalogService.h"
 
-#include "tep.h"
 #include <iomanip>
 #include <iostream>
 #include <map>
+#include "tdatablock.h"
 
 #include "tname.h"
 #include "ttypes.h"

source/libs/qworker/test/qworkerTests.cpp

@@ -33,7 +33,7 @@
 #include "taos.h"
 #include "tdef.h"
 #include "tvariant.h"
-#include "tep.h"
+#include "tdatablock.h"
 #include "trpc.h"
 #include "planner.h"
 #include "qworker.h"

source/libs/scalar/inc/filterInt.h

@@ -20,13 +20,13 @@
 extern "C" {
 #endif
 
+#include "query.h"
+#include "querynodes.h"
+#include "scalar.h"
+#include "tcommon.h"
+#include "tdatablock.h"
 #include "thash.h"
 #include "tname.h"
-#include "tcommon.h"
-#include "scalar.h"
-#include "querynodes.h"
-#include "query.h"
-#include "tep.h"
 
 #define FILTER_DEFAULT_GROUP_SIZE 4
 #define FILTER_DEFAULT_UNIT_SIZE 4

source/libs/scalar/src/filter.c

@@ -16,11 +16,11 @@
 #include <tlog.h>
 #include "thash.h"
 //#include "queryLog.h"
-#include "tcompare.h"
+#include "filter.h"
 #include "filterInt.h"
 #include "sclInt.h"
-#include "filter.h"
-#include "tep.h"
+#include "tcompare.h"
+#include "tdatablock.h"
 
 OptrStr gOptrStr[] = {
   {0,                                      "invalid"},

source/libs/scalar/src/scalar.c

-#include "nodes.h"
-#include "tcommon.h"
-#include "querynodes.h"
 #include "function.h"
 #include "functionMgt.h"
-#include "sclvector.h"
+#include "nodes.h"
+#include "querynodes.h"
 #include "sclInt.h"
-#include "tep.h"
+#include "sclvector.h"
+#include "tcommon.h"
+#include "tdatablock.h"
 
 int32_t scalarGetOperatorParamNum(EOperatorType type) {
   if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type 

source/libs/scalar/src/sclvector.c

@@ -15,15 +15,15 @@
 
 #include "os.h"
 
-#include "ttypes.h"
-#include "sclvector.h"
-#include "tcompare.h"
-#include "querynodes.h"
+#include "filter.h"
 #include "filterInt.h"
 #include "query.h"
+#include "querynodes.h"
 #include "sclInt.h"
-#include "tep.h"
-#include "filter.h"
+#include "sclvector.h"
+#include "tcompare.h"
+#include "tdatablock.h"
+#include "ttypes.h"
 
 //GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i]));                                
 

source/libs/scalar/test/filter/filterTests.cpp

@@ -33,12 +33,18 @@
 #include "taos.h"
 #include "tdef.h"
 #include "tvariant.h"
-#include "tep.h"
+#include "tdatablock.h"
 #include "stub.h"
 #include "scalar.h"
+#include "filter.h"
 #include "nodes.h"
+#include "scalar.h"
+#include "stub.h"
+#include "taos.h"
+#include "tdatablock.h"
+#include "tdef.h"
 #include "tlog.h"
-#include "filter.h"
+#include "tvariant.h"
 
 namespace {
 

source/libs/scalar/test/scalar/scalarTests.cpp

@@ -33,7 +33,7 @@
 #include "taos.h"
 #include "tdef.h"
 #include "tvariant.h"
-#include "tep.h"
+#include "tdatablock.h"
 #include "stub.h"
 #include "scalar.h"
 #include "nodes.h"

source/libs/scheduler/test/schedulerTests.cpp

@@ -34,8 +34,20 @@
 #include "tvariant.h"
 #include "catalog.h"
 #include "scheduler.h"
-#include "tep.h"
+#include "taos.h"
+#include "tdatablock.h"
+#include "tdef.h"
 #include "trpc.h"
+#include "tvariant.h"
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+#pragma GCC diagnostic ignored "-Wreturn-type"
+#pragma GCC diagnostic ignored "-Wformat"
+
 #include "schedulerInt.h"
 #include "stub.h"
 #include "tref.h"

source/libs/sync/src/syncIO.c

@@ -14,7 +14,7 @@
  */
 
 #include "syncIO.h"
-#include <tep.h>
+#include <tdatablock.h>
 #include "syncOnMessage.h"
 #include "tglobal.h"
 #include "ttimer.h"

source/libs/transport/test/pushClient.c

@@ -14,7 +14,7 @@
  */
 #include <sys/time.h>
 
-#include <tep.h>
+#include <tdatablock.h>
 #include "os.h"
 #include "rpcLog.h"
 #include "taoserror.h"

source/libs/transport/test/rclient.c

@@ -14,7 +14,7 @@
  */
 #include <sys/time.h>
 
-#include <tep.h>
+#include <tdatablock.h>
 #include "os.h"
 #include "rpcLog.h"
 #include "taoserror.h"

source/libs/transport/test/syncClient.c

@@ -14,7 +14,7 @@
  */
 #include <sys/time.h>
 
-#include <tep.h>
+#include <tdatablock.h>
 #include "os.h"
 #include "rpcLog.h"
 #include "taoserror.h"

source/libs/transport/test/transUT.cc

@@ -15,10 +15,10 @@
 #include <gtest/gtest.h>
 #include <cstdio>
 #include <cstring>
-#include "tep.h"
+#include "tdatablock.h"
 #include "tglobal.h"
-#include "trpc.h"
 #include "tlog.h"
+#include "trpc.h"
 using namespace std;
 
 const char *label = "APP";

source/util/src/thash.c

@@ -36,25 +36,35 @@
     tfree(_n);             \
   } while (0);
 
+struct SHashNode {
+  SHashNode        *next;
+  uint32_t          hashVal;  // the hash value of key
+  uint32_t          dataLen;  // length of data
+  uint32_t          keyLen;   // length of the key
+  uint16_t          refCount; // reference count
+  int8_t            removed;  // flag to indicate removed
+  char              data[];
+};
+
 typedef struct SHashEntry {
-  int32_t    num;      // number of elements in current entry
-  SRWLatch   latch;    // entry latch
-  SHashNode *next;
+  int32_t           num;      // number of elements in current entry
+  SRWLatch          latch;    // entry latch
+  SHashNode        *next;
 } SHashEntry;
 
-typedef struct SHashObj {
-  SHashEntry     **hashList;
-  size_t           capacity;     // number of slots
-  size_t           size;         // number of elements in hash table
-  _hash_fn_t       hashFp;       // hash function
-  _equal_fn_t      equalFp;      // equal function
-  _hash_free_fn_t  freeFp;       // hash node free callback function
-  SRWLatch         lock;         // read-write spin lock
-  SHashLockTypeE   type;         // lock type
-  bool             enableUpdate; // enable update
-  SArray          *pMemBlock;    // memory block allocated for SHashEntry
-  _hash_before_fn_t callbackFp;  // function invoked before return the value to caller
-} SHashObj;
+struct SHashObj {
+  SHashEntry **     hashList;
+  size_t            capacity;      // number of slots
+  size_t            size;          // number of elements in hash table
+  _hash_fn_t        hashFp;        // hash function
+  _equal_fn_t       equalFp;       // equal function
+  _hash_free_fn_t   freeFp;        // hash node free callback function
+  SRWLatch          lock;          // read-write spin lock
+  SHashLockTypeE    type;          // lock type
+  bool              enableUpdate;  // enable update
+  SArray *          pMemBlock;     // memory block allocated for SHashEntry
+  _hash_before_fn_t callbackFp;    // function invoked before return the value to caller
+};
 
 /*
  * Function definition
@@ -367,7 +377,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
 
     // enable resize
     taosHashRUnlock(pHashObj);
-    return pHashObj->enableUpdate ? 0 : -1;
+    return pHashObj->enableUpdate ? 0 : -2;
   }
 }
 
@@ -464,7 +474,7 @@ void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void**
   return data;
 }
 
-int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t dsize) {
+int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
   if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL || keyLen == 0) {
     return -1;
   }
@@ -507,8 +517,6 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
           prevNode->next = pNode->next;
         }
 
-        if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize);
-
         pe->num--;
         atomic_sub_fetch_64(&pHashObj->size, 1);
         FREE_HASH_NODE(pNode);
@@ -525,10 +533,6 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
   return code;
 }
 
-int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
-  return taosHashRemoveWithData(pHashObj, key, keyLen, NULL, 0);
-}
-
 void taosHashClear(SHashObj *pHashObj) {
   if (pHashObj == NULL) {
     return;