Merge pull request #22034 from taosdata/feat/TS-3239

feat/TS-3239/TS-3596: port 'keepTimeOffset' from 2.6 to 3.0

docs/en/14-reference/12-config/index.md

@@ -722,6 +722,16 @@ The charset that takes effect is UTF-8.
 | Value Range   | 0: not change; 1: change by modification                                      |
 | Default Value | 0                                                                             |
 
+### keepTimeOffset
+
+| Attribute     | Description               |
+| ------------- | ------------------------- |
+| Applicable    | Server Only               |
+| Meaning       | Latency of data migration |
+| Unit          | hour                      |
+| Value Range   | 0-23                      |
+| Default Value | 0                         |
+
 ## 3.0 Parameters
 
 | #   |     **Parameter**      | **Applicable to 2.x ** | **Applicable to  3.0 **      | Current behavior in 3.0 |
@@ -779,3 +789,4 @@ The charset that takes effect is UTF-8.
 | 53  |          udf           | Yes                    | Yes                          |                         |
 | 54  |     enableCoreFile     | Yes                    | Yes                          |                         |
 | 55  |    ttlChangeOnWrite    | No                     | Yes                          |                         |
+| 56  |     keepTimeOffset     | Yes                    | Yes                          |                         |

include/common/tglobal.h

@@ -48,6 +48,7 @@ extern int32_t tsMaxNumOfDistinctResults;
 extern int32_t tsCompatibleModel;
 extern bool    tsPrintAuth;
 extern int64_t tsTickPerMin[3];
+extern int64_t tsTickPerHour[3];
 extern int32_t tsCountAlwaysReturnValue;
 extern float   tsSelectivityRatio;
 extern int32_t tsTagFilterResCacheSize;
@@ -185,6 +186,7 @@ extern bool    tsDisableStream;
 extern int64_t tsStreamBufferSize;
 extern int64_t tsCheckpointInterval;
 extern bool    tsFilterScalarMode;
+extern int32_t tsKeepTimeOffset;
 extern int32_t tsMaxStreamBackendCache;
 extern int32_t tsPQSortMemThreshold;
 

packaging/cfg/taos.cfg

@@ -108,6 +108,9 @@
 # time period of keeping log files, in days
 # logKeepDays               0
 
+# unit Hour. Latency of data migration
+# keepTimeOffset            0
+
 
 ############ 3. Debug Flag and levels #############################################
 

source/common/src/tglobal.c

@@ -186,6 +186,13 @@ bool tsDeployOnSnode = true;
  *     TSDB_TIME_PRECISION_NANO:  60000000000L
  */
 int64_t tsTickPerMin[] = {60000L, 60000000L, 60000000000L};
+/*
+ * millisecond by default
+ * for TSDB_TIME_PRECISION_MILLI: 3600000L
+ *     TSDB_TIME_PRECISION_MICRO: 3600000000L
+ *     TSDB_TIME_PRECISION_NANO:  3600000000000L
+ */
+int64_t tsTickPerHour[] = {3600000L, 3600000000L, 3600000000000L};
 
 // lossy compress 6
 char tsLossyColumns[32] = "";  // "float|double" means all float and double columns can be lossy compressed.  set empty
@@ -217,6 +224,7 @@ bool    tsDisableStream = false;
 int64_t tsStreamBufferSize = 128 * 1024 * 1024;
 int64_t tsCheckpointInterval = 3 * 60 * 60 * 1000;
 bool    tsFilterScalarMode = false;
+int32_t tsKeepTimeOffset = 0;  // latency of data migration
 
 #ifndef _STORAGE
 int32_t taosSetTfsCfg(SConfig *pCfg) {
@@ -537,6 +545,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, 0) != 0) return -1;
 
   if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, 0) != 0) return -1;
+  if (cfgAddInt32(pCfg, "keepTimeOffset", tsKeepTimeOffset, 0, 23, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, 0) != 0) return -1;
 
@@ -921,6 +930,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsCheckpointInterval = cfgGetItem(pCfg, "checkpointInterval")->i64;
 
   tsFilterScalarMode = cfgGetItem(pCfg, "filterScalarMode")->bval;
+  tsKeepTimeOffset = cfgGetItem(pCfg, "keepTimeOffset")->i32;
   tsMaxStreamBackendCache = cfgGetItem(pCfg, "maxStreamBackendCache")->i32;
   tsPQSortMemThreshold = cfgGetItem(pCfg, "pqSortMemThreshold")->i32;
 
@@ -1478,6 +1488,19 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
     return;
   }
 
+  if (strcasecmp(option, "keepTimeOffset") == 0) {
+    int32_t newKeepTimeOffset = atoi(value);
+    if (newKeepTimeOffset < 0 || newKeepTimeOffset > 23) {
+      uError("failed to set keepTimeOffset from %d to %d. Valid range: [0, 23]", tsKeepTimeOffset, newKeepTimeOffset);
+      return;
+    }
+
+    uInfo("keepTimeOffset set from %d to %d", tsKeepTimeOffset, newKeepTimeOffset);
+    tsKeepTimeOffset = newKeepTimeOffset;
+
+    return;
+  }
+
   const char *options[] = {
       "dDebugFlag",   "vDebugFlag",   "mDebugFlag",   "wDebugFlag",    "sDebugFlag",   "tsdbDebugFlag", "tqDebugFlag",
       "fsDebugFlag",  "udfDebugFlag", "smaDebugFlag", "idxDebugFlag",  "tdbDebugFlag", "tmrDebugFlag",  "uDebugFlag",

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -542,6 +542,8 @@ int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t nowSec) {
     ASSERT(0);
   }
 
+  nowSec = nowSec - tsKeepTimeOffset * tsTickPerHour[pKeepCfg->precision];
+
   key = nowSec - pKeepCfg->keep0 * tsTickPerMin[pKeepCfg->precision];
   aFid[0] = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision);
   key = nowSec - pKeepCfg->keep1 * tsTickPerMin[pKeepCfg->precision];