[TD-32] refactor the tablemeta

src/client/CMakeLists.txt

@@ -6,6 +6,7 @@ INCLUDE_DIRECTORIES(jni)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
 INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 AUX_SOURCE_DIRECTORY(src SRC)

src/client/inc/tschemautil.h

@@ -89,17 +89,19 @@ bool isValidSchema(struct SSchema *pSchema, int32_t numOfCols);
  * get the schema for the "tbname" column. it is a built column
  * @return
  */
-SSchema tsGetTbnameColumnSchema();
+SSchema tscGetTbnameColumnSchema();
+
+/**
+ * create the table meta from the msg
+ * @param pTableMetaMsg
+ * @param size size of the table meta
+ * @return
+ */
+STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size);
 
 //todo tags value as well as the table id structure needs refactor
 char *tsGetTagsValue(STableMeta *pMeta);
 
-bool tsMeterMetaIdentical(STableMeta *p1, STableMeta *p2);
-
-void extractTableName(char *tableId, char *name);
-
-SSQLToken extractDBName(char *tableId, char *name);
-
 void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable);
 
 #ifdef __cplusplus

src/client/src/tscSQLParser.c

@@ -29,6 +29,8 @@
 #include "tsclient.h"
 #include "ttokendef.h"
 
+#include "name.h"
+
 #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0"
 
 // -1 is tbname column index, so here use the -2 as the initial value
@@ -2470,7 +2472,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
 
   STableMeta* pTableMeta = NULL;
   SSchema*    pSchema = NULL;
-  SSchema     s = tsGetTbnameColumnSchema();
+  SSchema     s = tscGetTbnameColumnSchema();
 
   int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL;
 
@@ -3538,8 +3540,9 @@ static int32_t setTableCondForMetricQuery(SQueryInfo* pQueryInfo, const char* ac
   }
   num = j;
 
-  SSQLToken dbToken = extractDBName(pTableMetaInfo->name, db);
-
+  char* name = extractDBName(pTableMetaInfo->name, db);
+  SSQLToken dbToken = {.type = TK_STRING, .z = name, .n = strlen(name)};
+  
   for (int32_t i = 0; i < num; ++i) {
     if (i >= 1) {
       taosStringBuilderAppendStringLen(&sb1, TBNAME_LIST_SEP, 1);

src/client/src/tscSchemaUtil.c

@@ -45,8 +45,6 @@ int32_t tscGetNumOfColumns(const STableMeta* pTableMeta) {
   
   // table created according to super table, use data from super table
   STableInfo tinfo = tscGetTableInfo(pTableMeta);
-  assert(tinfo.numOfColumns >= 2);
-  
   return tinfo.numOfColumns;
 }
 
@@ -141,7 +139,7 @@ SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t startCol)
   return &pSchema[startCol];
 }
 
-struct SSchema tsGetTbnameColumnSchema() {
+struct SSchema tscGetTbnameColumnSchema() {
   struct SSchema s = {
       .colId = TSDB_TBNAME_COLUMN_INDEX,
       .type  = TSDB_DATA_TYPE_BINARY,
@@ -152,6 +150,22 @@ struct SSchema tsGetTbnameColumnSchema() {
   return s;
 }
 
+STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) {
+  assert(pTableMetaMsg != NULL);
+  
+  int32_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema);
+  STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize);
+  
+  memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize);
+  
+  if (size != NULL) {
+    *size = sizeof(STableMeta) + schemaSize;
+  }
+  
+  return pTableMeta;
+}
+
+
 /**
  * the TableMeta data format in memory is as follows:
  *
@@ -174,26 +188,6 @@ char* tsGetTagsValue(STableMeta* pTableMeta) {
   return ((char*)pTableMeta + offset);
 }
 
-bool tsMeterMetaIdentical(STableMeta* p1, STableMeta* p2) {
-  if (p1 == NULL || p2 == NULL || p1->uid != p2->uid || p1->sversion != p2->sversion) {
-    return false;
-  }
-//
-//  if (p1 == p2) {
-//    return true;
-//  }
-//
-//  size_t size = sizeof(STableMeta) + p1->numOfColumns * sizeof(SSchema);
-//
-//  for (int32_t i = 0; i < p1->numOfTags; ++i) {
-//    SSchema* pColSchema = tscGetTableColumnSchema(p1, i + p1->numOfColumns);
-//    size += pColSchema->bytes;
-//  }
-
-//  return memcmp(p1, p2, size) == 0;
- return true;
-}
-
 // todo refactor
 static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) {
   for (int32_t i = 0; i < num; ++i) {
@@ -213,24 +207,6 @@ static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) {
   return len;
 }
 
-/**
- * extract table name from meterid, which the format of userid.dbname.metername
- * @param tableId
- * @return
- */
-void extractTableName(char* tableId, char* name) {
-  char* r = skipSegments(tableId, TS_PATH_DELIMITER[0], 2);
-  copy(name, r, TS_PATH_DELIMITER[0]);
-}
-
-SSQLToken extractDBName(char* tableId, char* name) {
-  char* r = skipSegments(tableId, TS_PATH_DELIMITER[0], 1);
-  size_t len = copy(name, r, TS_PATH_DELIMITER[0]);
-
-  SSQLToken token = {.z = name, .n = len, .type = TK_STRING};
-  return token;
-}
-
 /*
  * tablePrefix.columnName
  * extract table name and save it in pTable, with only column name in pToken

src/client/src/tscServer.c

@@ -2247,7 +2247,7 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   return TSDB_CODE_SUCCESS;
 }
 
-int tscBuildMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
+int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SCMTableInfoMsg *pInfoMsg;
   char *         pMsg;
   int            msgLen = 0;
@@ -2878,7 +2878,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
 
-  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);  //?
+  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
 
   STableMetaInfo *pTableMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0);
 
@@ -2891,7 +2891,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
 
   pMetaMsg->numOfColumns = ntohs(pMetaMsg->numOfColumns);
 
-  pSchema = (SSchema *)((char *)pMetaMsg + sizeof(STableMeta));
+  pSchema = pMetaMsg->schema;
   pMetaMsg->sid = ntohs(pMetaMsg->sid);
   for (int i = 0; i < pMetaMsg->numOfColumns; ++i) {
     pSchema->bytes = htons(pSchema->bytes);
@@ -2902,12 +2902,14 @@ int tscProcessShowRsp(SSqlObj *pSql) {
   strcpy(key + 1, "showlist");
 
   taosCacheRelease(tscCacheHandle, (void *)&(pTableMetaInfo->pTableMeta), false);
-
-  int32_t size = pMetaMsg->numOfColumns * sizeof(SSchema) + sizeof(STableMeta);
+  size_t size = 0;
+  STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size);
+  
   pTableMetaInfo->pTableMeta =
-      (STableMeta *)taosCachePut(tscCacheHandle, key, (char *)pMetaMsg, size, tsMeterMetaKeepTimer);
+      (STableMeta *)taosCachePut(tscCacheHandle, key, (char *)pTableMeta, size, tsMeterMetaKeepTimer);
+  
   pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols;
-  SSchema *pMeterSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
+  SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
 
   tscColumnBaseInfoReserve(&pQueryInfo->colList, pMetaMsg->numOfColumns);
   SColumnIndex index = {0};
@@ -2915,13 +2917,15 @@ int tscProcessShowRsp(SSqlObj *pSql) {
   for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i) {
     index.columnIndex = i;
     tscColumnBaseInfoInsert(pQueryInfo, &index);
-    tscFieldInfoSetValFromSchema(&pQueryInfo->fieldsInfo, i, &pMeterSchema[i]);
+    tscFieldInfoSetValFromSchema(&pQueryInfo->fieldsInfo, i, &pTableSchema[i]);
     
     pQueryInfo->fieldsInfo.pSqlExpr[i] = tscSqlExprInsert(pQueryInfo, i, TSDB_FUNC_TS_DUMMY, &index,
-                     pMeterSchema[i].type, pMeterSchema[i].bytes, pMeterSchema[i].bytes);
+                     pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes);
   }
 
   tscFieldInfoCalOffset(pQueryInfo);
+  
+  tfree(pTableMeta);
   return 0;
 }
 
@@ -3379,7 +3383,7 @@ void tscInitMsgs() {
 
   tscBuildMsg[TSDB_SQL_CONNECT] = tscBuildConnectMsg;
   tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg;
-  tscBuildMsg[TSDB_SQL_META] = tscBuildMeterMetaMsg;
+  tscBuildMsg[TSDB_SQL_META] = tscBuildTableMetaMsg;
   tscBuildMsg[TSDB_SQL_METRIC] = tscBuildMetricMetaMsg;
   tscBuildMsg[TSDB_SQL_MULTI_META] = tscBuildMultiMeterMetaMsg;
 

src/common/inc/name.h

+#ifndef TDENGINE_NAME_H
+#define TDENGINE_NAME_H
+
+int32_t extractTableName(const char *tableId, char *name);
+
+char* extractDBName(const char *tableId, char *name);
+
+#endif  // TDENGINE_NAME_H

src/common/src/name.c

+#include "os.h"
+#include "tutil.h"
+
+#include "name.h"
+#include "tstoken.h"
+#include "ttokendef.h"
+
+// todo refactor
+static FORCE_INLINE const char* skipSegments(const char* input, char delim, int32_t num) {
+  for (int32_t i = 0; i < num; ++i) {
+    while (*input != 0 && *input++ != delim) {
+    };
+  }
+  return input;
+}
+
+static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) {
+  size_t len = 0;
+  while (*src != delimiter && *src != 0) {
+    *dst++ = *src++;
+    len++;
+  }
+  
+  return len;
+}
+
+int32_t extractTableName(const char* tableId, char* name) {
+  size_t offset = strcspn(tableId, &TS_PATH_DELIMITER[0]);
+  offset = strcspn(&tableId[offset], &TS_PATH_DELIMITER[0]);
+  
+  return strncpy(name, &tableId[offset], TSDB_TABLE_NAME_LEN);
+  
+//  char* r = skipSegments(tableId, TS_PATH_DELIMITER[0], 2);
+//  return copy(name, r, TS_PATH_DELIMITER[0]);
+}
+
+char* extractDBName(const char* tableId, char* name) {
+  size_t offset1 = strcspn(tableId, &TS_PATH_DELIMITER[0]);
+  size_t len = strcspn(&tableId[offset1 + 1], &TS_PATH_DELIMITER[0]);
+  
+  return strncpy(name, &tableId[offset1 + 1], len);
+}

src/inc/taosmsg.h

@@ -223,7 +223,7 @@ typedef struct {
 
 typedef struct SSchema {
   uint8_t type;
-  char    name[TSDB_COL_NAME_LEN + 1];
+  char    name[TSDB_COL_NAME_LEN];
   int16_t colId;
   int16_t bytes;
 } SSchema;
@@ -684,10 +684,9 @@ typedef struct STableMetaMsg {
   uint8_t numOfTags;
   uint8_t precision;
   uint8_t tableType;
-//  uint8_t index : 4;  // used locally
   int16_t numOfColumns;
-//  int16_t rowSize;  // used locally, calculated in client
   int16_t sversion;
+  
   int8_t  numOfVpeers;
   SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT];
   int32_t  sid;

src/mnode/src/mgmtDb.c

@@ -15,23 +15,26 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "tstatus.h"
-#include "tutil.h"
-#include "mnode.h"
+
+#include "mgmtDb.h"
 #include "mgmtAcct.h"
 #include "mgmtBalance.h"
-#include "mgmtDb.h"
+#include "mgmtChildTable.h"
 #include "mgmtDnode.h"
-#include "mgmtMnode.h"
 #include "mgmtGrant.h"
-#include "mgmtShell.h"
+#include "mgmtMnode.h"
 #include "mgmtNormalTable.h"
-#include "mgmtChildTable.h"
+#include "mgmtShell.h"
 #include "mgmtSuperTable.h"
 #include "mgmtTable.h"
 #include "mgmtUser.h"
 #include "mgmtVgroup.h"
+#include "mnode.h"
+
+#include "taoserror.h"
+#include "tstatus.h"
+#include "tutil.h"
+#include "name.h"
 
 static void *tsDbSdb = NULL;
 static int32_t tsDbUpdateSize;
@@ -294,8 +297,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
 
 bool mgmtCheckIsMonitorDB(char *db, char *monitordb) {
   char dbName[TSDB_DB_NAME_LEN + 1] = {0};
-  assert(0);
-//  extractDBName(db, dbName);
+  extractDBName(db, dbName);
 
   size_t len = strlen(dbName);
   return (strncasecmp(dbName, monitordb, len) == 0 && len == strlen(monitordb));
@@ -436,7 +438,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
-  strcpy(pSchema[cols].name, "created time");
+  strcpy(pSchema[cols].name, "created_time");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 

src/mnode/src/mgmtDnode.c

@@ -331,7 +331,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
 
   pShow->bytes[cols] = 12;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "sync status");
+  strcpy(pSchema[cols].name, "sync_status");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 

src/mnode/src/mgmtSuperTable.c

@@ -508,7 +508,7 @@ static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow,
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
-  strcpy(pSchema[cols].name, "create time");
+  strcpy(pSchema[cols].name, "create_time");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 

src/mnode/src/mgmtTable.c

@@ -198,13 +198,13 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
 
   pShow->bytes[cols] = TSDB_TABLE_NAME_LEN;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "table name");
+  strcpy(pSchema[cols].name, "table_name");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
-  strcpy(pSchema[cols].name, "create time");
+  strcpy(pSchema[cols].name, "created_time");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
@@ -216,7 +216,7 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
 
   pShow->bytes[cols] = TSDB_TABLE_NAME_LEN;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "super table name");
+  strcpy(pSchema[cols].name, "stable_name");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 

src/mnode/src/mgmtUser.c

@@ -193,7 +193,7 @@ static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCon
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
-  strcpy(pSchema[cols].name, "created time");
+  strcpy(pSchema[cols].name, "created_time");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 

src/mnode/src/mgmtVgroup.c

@@ -233,7 +233,7 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
 
   pShow->bytes[cols] = 9;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "vgroup status");
+  strcpy(pSchema[cols].name, "vgroup_status");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
@@ -273,13 +273,13 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
 
     pShow->bytes[cols] = 9;
     pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-    strcpy(pSchema[cols].name, "vnode status");
+    strcpy(pSchema[cols].name, "vnode_status");
     pSchema[cols].bytes = htons(pShow->bytes[cols]);
     cols++;
 
     pShow->bytes[cols] = 16;
     pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-    strcpy(pSchema[cols].name, "public ip");
+    strcpy(pSchema[cols].name, "public_ip");
     pSchema[cols].bytes = htons(pShow->bytes[cols]);
     cols++;
   }

src/query/CMakeLists.txt

@@ -9,5 +9,5 @@ INCLUDE_DIRECTORIES(inc)
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(query ${SRC})
-  TARGET_LINK_LIBRARIES(query util m rt)
+  TARGET_LINK_LIBRARIES(query tutil m rt)
 ENDIF ()
\ No newline at end of file

src/util/src/tcache.c

@@ -206,9 +206,10 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pObj, SCacheDataNode *p
     return;
   }
   
+  int32_t size = pNode->size;
   taosHashRemove(pObj->pHashTable, pNode->key, pNode->keySize);
-  pTrace("key:%s is removed from cache,total:%d,size:%ldbytes", pNode->key, pObj->totalSize, pObj->totalSize);
   
+  pTrace("key:%s is removed from cache,total:%d,size:%ldbytes", pNode->key, pObj->totalSize, size);
   free(pNode);
 }
 
@@ -418,10 +419,10 @@ void *taosCachePut(void *handle, char *key, char *pData, int dataSize, int durat
   if (pOld == NULL) {  // do addedTime to cache
     pNode = taosAddToCacheImpl(pObj, key, keyLen, pData, dataSize, duration * 1000L);
     if (NULL != pNode) {
-      pTrace("key:%s %p added into cache, addedTime:%" PRIu64 ", expireTime:%" PRIu64 ", cache total:%d, size:%" PRId64
-                 " bytes, collision:%d",
-             key, pNode, pNode->addedTime, pNode->expiredTime, dataSize, pObj->totalSize,
-             pObj->statistics.numOfCollision);
+      pObj->totalSize += pNode->size;
+      
+      pTrace("key:%s %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%d, size:%" PRId64 " bytes",
+             key, pNode, pNode->addedTime, pNode->expiredTime, pObj->totalSize, dataSize);
     }
   } else {  // old data exists, update the node
     pNode = taosUpdateCacheImpl(pObj, pOld, key, keyLen, pData, dataSize, duration * 1000L);