Merge branch '3.0' into feature/3.0_interval_hash_optimize

include/common/tmsg.h

@@ -1201,6 +1201,7 @@ typedef struct {
   int16_t  sstTrigger;
   int16_t  hashPrefix;
   int16_t  hashSuffix;
+  int32_t  tsdbPageSize;
 } SCreateVnodeReq;
 
 int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq);

source/common/src/systable.c

@@ -102,9 +102,10 @@ static const SSysDbTableSchema userDBSchema[] = {
     {.name = "wal_retention_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true},
     {.name = "wal_roll_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
     {.name = "wal_segment_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true},
-    {.name = "sst_trigger", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
+    {.name = "stt_trigger", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
     {.name = "table_prefix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
     {.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
+    {.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
 };
 
 static const SSysDbTableSchema userFuncSchema[] = {

source/common/src/tmsg.c

@@ -3782,6 +3782,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
   if (tEncodeI16(&encoder, pReq->sstTrigger) < 0) return -1;
   if (tEncodeI16(&encoder, pReq->hashPrefix) < 0) return -1;
   if (tEncodeI16(&encoder, pReq->hashSuffix) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1;
 
   tEndEncode(&encoder);
 
@@ -3857,6 +3858,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
   if (tDecodeI16(&decoder, &pReq->sstTrigger) < 0) return -1;
   if (tDecodeI16(&decoder, &pReq->hashPrefix) < 0) return -1;
   if (tDecodeI16(&decoder, &pReq->hashSuffix) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->tsdbPageSize) < 0) return -1;
 
   tEndDecode(&decoder);
   tDecoderClear(&decoder);

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -173,6 +173,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->hashMethod = pCreate->hashMethod;
   pCfg->hashPrefix = pCreate->hashPrefix;
   pCfg->hashSuffix = pCreate->hashSuffix;
+  pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024;
 
   pCfg->standby = pCfg->standby;
   pCfg->syncCfg.myIndex = pCreate->selfIndex;
@@ -222,9 +223,11 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     return -1;
   }
 
-  dInfo("vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d sstTrigger:%d",
+  dInfo(
+      "vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d sstTrigger:%d "
+      "tsdbPageSize:%d",
       createReq.vgId, createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize,
-        createReq.sstTrigger);
+      createReq.sstTrigger, createReq.tsdbPageSize);
   dInfo("vgId:%d, hashMethod:%d begin:%u end:%u prefix:%d surfix:%d", createReq.vgId, createReq.hashMethod,
         createReq.hashBegin, createReq.hashEnd, createReq.hashPrefix, createReq.hashSuffix);
   vmGenerateVnodeCfg(&createReq, &vnodeCfg);

source/dnode/mnode/impl/inc/mndDef.h

@@ -308,6 +308,7 @@ typedef struct {
   int16_t hashPrefix;
   int16_t hashSuffix;
   int16_t sstTrigger;
+  int32_t tsdbPageSize;
   int32_t numOfRetensions;
   SArray* pRetensions;
   int32_t walRetentionPeriod;

source/dnode/mnode/impl/src/mndDb.c

@@ -31,7 +31,7 @@
 #include "systable.h"
 
 #define DB_VER_NUMBER   1
-#define DB_RESERVE_SIZE 58
+#define DB_RESERVE_SIZE 54
 
 static SSdbRaw *mndDbActionEncode(SDbObj *pDb);
 static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
@@ -128,6 +128,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
   SDB_SET_INT16(pRaw, dataPos, pDb->cfg.sstTrigger, _OVER)
   SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashPrefix, _OVER)
   SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashSuffix, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.tsdbPageSize, _OVER)
 
   SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
   SDB_SET_DATALEN(pRaw, dataPos, _OVER)
@@ -214,10 +215,20 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.sstTrigger, _OVER)
   SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashPrefix, _OVER)
   SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashSuffix, _OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.tsdbPageSize, _OVER)
 
   SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
   taosInitRWLatch(&pDb->lock);
 
+  if (pDb->cfg.tsdbPageSize <= TSDB_MIN_TSDB_PAGESIZE) {
+    mInfo("db:%s, tsdbPageSize set from %d to default %d", pDb->name, pDb->cfg.tsdbPageSize,
+          TSDB_DEFAULT_TSDB_PAGESIZE);
+  }
+
+  if (pDb->cfg.sstTrigger <= TSDB_MIN_STT_TRIGGER) {
+    mInfo("db:%s, sstTrigger set from %d to default %d", pDb->name, pDb->cfg.sstTrigger, TSDB_DEFAULT_SST_TRIGGER);
+  }
+
   terrno = 0;
 
 _OVER:
@@ -262,6 +273,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
   pOld->cfg.cacheLast = pNew->cfg.cacheLast;
   pOld->cfg.replications = pNew->cfg.replications;
   pOld->cfg.sstTrigger = pNew->cfg.sstTrigger;
+  pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize;
   taosWUnLockLatch(&pOld->lock);
   return 0;
 }
@@ -341,6 +353,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
   if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1;
   if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1;
+  if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1;
 
   terrno = 0;
   return terrno;
@@ -377,6 +390,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER;
   if (pCfg->hashPrefix < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX;
   if (pCfg->hashSuffix < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX;
+  if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE;
 }
 
 static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
@@ -496,6 +510,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
       .sstTrigger = pCreate->sstTrigger,
       .hashPrefix = pCreate->hashPrefix,
       .hashSuffix = pCreate->hashSuffix,
+      .tsdbPageSize = pCreate->tsdbPageSize,
   };
 
   dbObj.cfg.numOfRetensions = pCreate->numOfRetensions;
@@ -1726,6 +1741,9 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.hashSuffix, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.tsdbPageSize, false);
   }
 
   taosMemoryFree(buf);

source/dnode/mnode/impl/src/mndVgroup.c

@@ -237,6 +237,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
   createReq.sstTrigger = pDb->cfg.sstTrigger;
   createReq.hashPrefix = pDb->cfg.hashPrefix;
   createReq.hashSuffix = pDb->cfg.hashSuffix;
+  createReq.tsdbPageSize = pDb->cfg.tsdbPageSize;
 
   for (int32_t v = 0; v < pVgroup->replica; ++v) {
     SReplica  *pReplica = &createReq.replicas[v];

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -115,6 +115,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
   if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize) < 0) return -1;
 
   if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1;
@@ -215,7 +216,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
   if (code < 0) return -1;
   tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code);
-  if (code < 0) return -1;
+  if (code < 0) pCfg->sttTrigger = TSDB_DEFAULT_SST_TRIGGER;
   tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code);
   if (code < 0) return -1;
   tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd, code);
@@ -223,9 +224,9 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod, code);
   if (code < 0) return -1;
   tjsonGetNumberValue(pJson, "hashPrefix", pCfg->hashPrefix, code);
-  if (code < 0) return -1;
+  if (code < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX;
   tjsonGetNumberValue(pJson, "hashSuffix", pCfg->hashSuffix, code);
-  if (code < 0) return -1;
+  if (code < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX;
 
   tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code);
   if (code < 0) return -1;
@@ -255,6 +256,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   }
 
   tjsonGetNumberValue(pJson, "tsdbPageSize", pCfg->tsdbPageSize, code);
+  if (code < 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE * 1024;
 
   return 0;
 }

source/libs/tdb/src/db/tdbPager.c

@@ -185,10 +185,10 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
   // ref page one more time so the page will not be release
   tdbRefPage(pPage);
   tdbDebug("pcache/mdirty page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id);
-  /*
+
   // Set page as dirty
   pPage->isDirty = 1;
-
+  /*
   // Add page to dirty list(TODO: NOT use O(n^2) algorithm)
   for (ppPage = &pPager->pDirty; (*ppPage) && TDB_PAGE_PGNO(*ppPage) < TDB_PAGE_PGNO(pPage);
        ppPage = &((*ppPage)->pDirtyNext)) {
@@ -260,6 +260,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
 
     pPage->isDirty = 0;
 
+    // tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage);
     tdbPCacheRelease(pPager->pCache, pPage, pTxn);
   }
 
@@ -345,15 +346,19 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
   }
   */
   // 3, release the dirty pages
-  for (pPage = pPager->pDirty; pPage; pPage = pPager->pDirty) {
-    pPager->pDirty = pPage->pDirtyNext;
-    pPage->pDirtyNext = NULL;
+  SRBTreeIter  iter = tRBTreeIterCreate(&pPager->rbt, 1);
+  SRBTreeNode *pNode = NULL;
+  while ((pNode = tRBTreeIterNext(&iter)) != NULL) {
+    pPage = (SPage *)pNode;
 
     pPage->isDirty = 0;
 
+    // tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage);
     tdbPCacheRelease(pPager->pCache, pPage, pTxn);
   }
 
+  tRBTreeCreate(&pPager->rbt, pageCmpFn);
+
   // 4, remove the journal file
   tdbOsClose(pPager->jfd);
   tdbOsRemove(pPager->jFileName);

tests/system-test/1-insert/delete_data.py

@@ -81,6 +81,7 @@ class TDTestCase:
             'binary':self.binary_val,
             'nchar':self.nchar_val
                     }
+        
     def insert_base_data(self,col_type,tbname,rows,base_data):
         for i in range(rows):
             if col_type.lower() == 'tinyint':
@@ -290,6 +291,9 @@ class TDTestCase:
         self.delete_data_ntb()
         self.delete_data_ctb()
         self.delete_data_stb()
+        tdDnodes.stoptaosd(1)
+        tdDnodes.starttaosd(1)
+        self.delete_data_ntb()
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/system-test/1-insert/update_data.py

@@ -154,10 +154,12 @@ class TDTestCase:
         up_bool = random.randint(0,100)%2
         up_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX)
         up_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300))
-        binary_length = random.randint(0,self.str_length)
-        nchar_length = random.randint(0,self.str_length)
-        up_binary = tdCom.getLongName(binary_length)
-        up_nchar = tdCom.getLongName(nchar_length)
+        binary_length = []
+        for i in range(self.str_length+1):
+            binary_length.append(i)
+        nchar_length = []
+        for i in range(self.str_length+1):
+            nchar_length.append(i)
         for col_name,col_type in column_dict.items():
             if tb_type == 'ntb':
                 tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})')
@@ -188,8 +190,12 @@ class TDTestCase:
             elif col_type.lower() == 'double':
                 self.update_and_check_data(tbname,col_name,col_type,up_double,dbname)
             elif 'binary' in col_type.lower():
+                for i in binary_length:
+                    up_binary = tdCom.getLongName(i)
                     self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname)
             elif 'nchar' in col_type.lower():
+                for i in nchar_length:
+                    up_nchar = tdCom.getLongName(i)
                     self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname)
             elif col_type.lower() == 'timestamp':
                 self.update_and_check_data(tbname,col_name,col_type,self.ts+1,dbname)