Merge branch '3.0' into fix/TD-21457-3.0

a171c0c8 · kailixu · a3bf49d2 · 0bb468af · a171c0c8 · a171c0c8
11 changed file
--- a/docs/en/13-operation/10-monitor.md
+++ b/docs/en/13-operation/10-monitor.md
@@ -36,7 +36,7 @@ You can use below command to setup Grafana alert notification.
 An existing Grafana Notification Channel can be specified with parameter `-E`, the notifier uid of the channel can be obtained by `curl -u admin:admin localhost:3000/api/alert-notifications |jq`

     ```bash
-     sudo ./TDinsight.sh -a http://localhost:6041 -u root -p taosdata -E <notifier uid>
+     ./TDinsight.sh -a http://localhost:6041 -u root -p taosdata -E <notifier uid>
     ```

 Launch `TDinsight.sh` with the command above and restart Grafana, then open Dashboard `http://localhost:3000/d/tdinsight`.
@@ -274,7 +274,7 @@ The data of tdinsight dashboard is stored in `log` database (default. You can ch
 |field|type|is\_tag|comment|
 |:----|:---|:-----|:------|
 |\_ts|TIMESTAMP||timestamp|
-|guage|DOUBLE||metric value|
+|gauge|DOUBLE||metric value|
 |client\_ip|NCHAR|TAG|client ip|
 |endpoint|NCHAR|TAG|taosadpater endpoint|
 |request\_method|NCHAR|TAG|request method|
@@ -288,7 +288,7 @@ The data of tdinsight dashboard is stored in `log` database (default. You can ch
 |field|type|is\_tag|comment|
 |:----|:---|:-----|:------|
 |\_ts|TIMESTAMP||timestamp|
-|guage|DOUBLE||metric value|
+|gauge|DOUBLE||metric value|
 |client\_ip|NCHAR|TAG|client ip|
 |endpoint|NCHAR|TAG|taosadpater endpoint|
 |request\_method|NCHAR|TAG|request method|
@@ -302,7 +302,7 @@ The data of tdinsight dashboard is stored in `log` database (default. You can ch
 |field|type|is\_tag|comment|
 |:----|:---|:-----|:------|
 |\_ts|TIMESTAMP||timestamp|
-|guage|DOUBLE||metric value|
+|gauge|DOUBLE||metric value|
 |endpoint|NCHAR|TAG|taosadpater endpoint|

 ### taosadapter\_restful\_http\_request\_summary\_milliseconds table
@@ -330,7 +330,7 @@ The data of tdinsight dashboard is stored in `log` database (default. You can ch
 |field|type|is\_tag|comment|
 |:----|:---|:-----|:------|
 |\_ts|TIMESTAMP||timestamp|
-|guage|DOUBLE||metric value|
+|gauge|DOUBLE||metric value|
 |endpoint|NCHAR|TAG|taosadpater endpoint|

 ### taosadapter\_system\_cpu\_percent table 
@@ -340,6 +340,6 @@ The data of tdinsight dashboard is stored in `log` database (default. You can ch
 |field|type|is\_tag|comment|
 |:----|:---|:-----|:------|
 |\_ts|TIMESTAMP||timestamp|
-|guage|DOUBLE||mertic value|
+|gauge|DOUBLE||mertic value|
 |endpoint|NCHAR|TAG|taosadpater endpoint|

--- a/docs/en/14-reference/07-tdinsight/index.md
+++ b/docs/en/14-reference/07-tdinsight/index.md
@@ -149,7 +149,7 @@ curl --no-progress-meter -u admin:admin http://localhost:3000/api/alert-notifica
 Use the `uid` value obtained above as `-E` input.

 ```bash
-sudo ./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord -E existing-notifier
+./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord -E existing-notifier
 ```

 If you want to monitor multiple TDengine clusters, you need to set up numerous TDinsight dashboards. Setting up non-default TDinsight requires some changes: the `-n` `-i` `-t` options need to be changed to non-default names, and `-N` and `-L` should also be changed if using the built-in SMS alerting feature.

--- a/docs/en/14-reference/12-config/index.md
+++ b/docs/en/14-reference/12-config/index.md
--- a/docs/zh/14-reference/07-tdinsight/index.mdx
+++ b/docs/zh/14-reference/07-tdinsight/index.mdx
@@ -177,7 +177,7 @@ Install and configure TDinsight dashboard in Grafana on Ubuntu 18.04/20.04 syste
 假设您在主机 `tdengine` 上启动 TDengine 数据库，HTTP API 端口为 `6041`，用户为 `root1`，密码为 `pass5ord`。执行脚本：

 ```bash
-sudo ./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord
+./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord
 ```

 我们提供了一个“-E”选项，用于从命令行配置 TDinsight 使用现有的通知通道(Notification Channel)。假设你的 Grafana 用户和密码是 `admin:admin`，使用以下命令获取已有的通知通道的`uid`：
@@ -189,7 +189,7 @@ curl --no-progress-meter -u admin:admin http://localhost:3000/api/alert-notifica
 使用上面获取的 `uid` 值作为 `-E` 输入。

 ```bash
-sudo ./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord -E existing-notifier
+./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord -E existing-notifier
 ```

 如果要监控多个 TDengine 集群，则需要设置多个 TDinsight 仪表盘。设置非默认 TDinsight 需要进行一些更改： `-n` `-i` `-t` 选项需要更改为非默认名称，如果使用 内置短信告警功能，`-N` 和 `-L` 也应该改变。

--- a/docs/zh/14-reference/12-config/index.md
+++ b/docs/zh/14-reference/12-config/index.md
--- a/docs/zh/17-operation/10-monitor.md
+++ b/docs/zh/17-operation/10-monitor.md
@@ -32,7 +32,7 @@ chmod +x TDinsight.sh
   - 使用已经存在的 Grafana Notification Channel `uid`，参数 `-E`。该参数可以使用 `curl -u admin:admin localhost:3000/api/alert-notifications |jq` 来获取。

     ```bash
-     sudo ./TDinsight.sh -a http://localhost:6041 -u root -p taosdata -E <notifier uid>
+     ./TDinsight.sh -a http://localhost:6041 -u root -p taosdata -E <notifier uid>
     ```

 运行程序并重启 Grafana 服务，打开面板：`http://localhost:3000/d/tdinsight`。
@@ -270,7 +270,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |field|type|is\_tag|comment|
 |:----|:---|:-----|:------|
 |\_ts|TIMESTAMP||timestamp|
-|guage|DOUBLE||监控指标值|
+|gauge|DOUBLE||监控指标值|
 |client\_ip|NCHAR|TAG|client ip|
 |endpoint|NCHAR|TAG|taosadpater endpoint|
 |request\_method|NCHAR|TAG|request method|
@@ -284,7 +284,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |field|type|is\_tag|comment|
 |:----|:---|:-----|:------|
 |\_ts|TIMESTAMP||timestamp|
-|guage|DOUBLE||监控指标值|
+|gauge|DOUBLE||监控指标值|
 |client\_ip|NCHAR|TAG|client ip|
 |endpoint|NCHAR|TAG|taosadpater endpoint|
 |request\_method|NCHAR|TAG|request method|
@@ -298,7 +298,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |field|type|is\_tag|comment|
 |:----|:---|:-----|:------|
 |\_ts|TIMESTAMP||timestamp|
-|guage|DOUBLE||监控指标值|
+|gauge|DOUBLE||监控指标值|
 |endpoint|NCHAR|TAG|taosadpater endpoint|

 ### taosadapter\_restful\_http\_request\_summary\_milliseconds 表
@@ -326,7 +326,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |field|type|is\_tag|comment|
 |:----|:---|:-----|:------|
 |\_ts|TIMESTAMP||timestamp|
-|guage|DOUBLE||监控指标值|
+|gauge|DOUBLE||监控指标值|
 |endpoint|NCHAR|TAG|taosadpater endpoint|

 ### taosadapter\_system\_cpu\_percent 表
@@ -336,5 +336,5 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |field|type|is\_tag|comment|
 |:----|:---|:-----|:------|
 |\_ts|TIMESTAMP||timestamp|
-|guage|DOUBLE||监控指标值|
+|gauge|DOUBLE||监控指标值|
 |endpoint|NCHAR|TAG|taosadpater endpoint|
--- a/docs/zh/27-train-faq/01-faq.md
+++ b/docs/zh/27-train-faq/01-faq.md
@@ -247,4 +247,10 @@ launchctl limit maxfiles
 该提示是创建 db 的 vnode 数量不够了，需要的 vnode 不能超过了 dnode 中 vnode 的上限。因为系统默认是一个 dnode 中有 CPU 核数两倍的 vnode，也可以通过配置文件中的参数 supportVnodes 控制。
 正常调大 taos.cfg 中 supportVnodes 参数即可。

+### 21 【查询】在服务器上的使用 tao-CLI 能查到指定时间段的数据，但在客户端机器上查不到？
+这种情况是因为客户端与服务器上设置的时区不一致导致的，调整客户端与服务器的时区一致即可解决。
+
+### 22 【表名】表名确认是存在的，但写入或查询时报表不存在错误，非常奇怪，什么原因？ 
+TDengine 中的所有名称，包括数据库名、表名等都是区分大小写的，如果这些名称在程序或 taos-CLI 中没有使用反引号（`）括起来使用，即使你输入的是大写的，引擎也会转化成小写来使用，如果名称前后加上了反引号，引擎就不会再转化成小写，会保持原样来使用。
+

--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -351,6 +351,7 @@ typedef struct {
  rocksdb_writeoptions_t              *writeoptions;
  rocksdb_readoptions_t               *readoptions;
  rocksdb_writebatch_t                *writebatch;
+  rocksdb_writebatch_t                *rwritebatch;
  TdThreadMutex                        rMutex;
  STSchema                            *pTSchema;
 } SRocksCache;

--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -164,8 +164,10 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) {
  }

  rocksdb_writebatch_t *writebatch = rocksdb_writebatch_create();
+  rocksdb_writebatch_t *rwritebatch = rocksdb_writebatch_create();

  pTsdb->rCache.writebatch = writebatch;
+  pTsdb->rCache.rwritebatch = rwritebatch;
  pTsdb->rCache.my_comparator = cmp;
  pTsdb->rCache.options = options;
  pTsdb->rCache.writeoptions = writeoptions;
@@ -198,6 +200,7 @@ static void tsdbCloseRocksCache(STsdb *pTsdb) {
  rocksdb_close(pTsdb->rCache.db);
  rocksdb_flushoptions_destroy(pTsdb->rCache.flushoptions);
  rocksdb_writebatch_destroy(pTsdb->rCache.writebatch);
+  rocksdb_writebatch_destroy(pTsdb->rCache.rwritebatch);
  rocksdb_readoptions_destroy(pTsdb->rCache.readoptions);
  rocksdb_writeoptions_destroy(pTsdb->rCache.writeoptions);
  rocksdb_options_destroy(pTsdb->rCache.options);
@@ -208,14 +211,18 @@ static void tsdbCloseRocksCache(STsdb *pTsdb) {
  taosMemoryFree(pTsdb->rCache.pTSchema);
 }

-static void rocksMayWrite(STsdb *pTsdb, bool force) {
+static void rocksMayWrite(STsdb *pTsdb, bool force, bool read) {
  rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch;
-
-  if (force || rocksdb_writebatch_count(wb) >= 1024) {
+  if (read) {
+    wb = pTsdb->rCache.rwritebatch;
+  }
+  int count = rocksdb_writebatch_count(wb);
+  if ((force && count > 0) || count >= 1024) {
    char *err = NULL;
    rocksdb_write(pTsdb->rCache.db, pTsdb->rCache.writeoptions, wb, &err);
    if (NULL != err) {
-      tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err);
+      tsdbError("vgId:%d, %s failed at line %d, count: %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, count,
+                err);
      rocksdb_free(err);
    }

@@ -227,7 +234,8 @@ int32_t tsdbCacheCommit(STsdb *pTsdb) {
  int32_t code = 0;
  char   *err = NULL;

-  rocksMayWrite(pTsdb, true);
+  rocksMayWrite(pTsdb, true, false);
+  rocksMayWrite(pTsdb, true, true);
  rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err);
  if (NULL != err) {
    tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err);
@@ -453,7 +461,7 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
  taosMemoryFree(values_list);
  taosMemoryFree(values_list_sizes);

-  rocksMayWrite(pTsdb, false);
+  rocksMayWrite(pTsdb, false, false);
  taosThreadMutexUnlock(&pTsdb->rCache.rMutex);

 _exit:
@@ -537,7 +545,7 @@ int32_t tsdbCacheGetSlow(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheR
        }

        // store result back to rocks cache
-        wb = pTsdb->rCache.writebatch;
+        wb = pTsdb->rCache.rwritebatch;
        char  *value = NULL;
        size_t vlen = 0;
        tsdbCacheSerialize(pLastCol, &value, &vlen);
@@ -552,7 +560,7 @@ int32_t tsdbCacheGetSlow(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheR
      }

      if (wb) {
-        rocksMayWrite(pTsdb, false);
+        rocksMayWrite(pTsdb, false, true);
      }

      taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
@@ -602,7 +610,7 @@ static SLastCol *tsdbCacheLoadCol(STsdb *pTsdb, SCacheRowsReader *pr, int16_t sl
      }

      // store result back to rocks cache
-      wb = pTsdb->rCache.writebatch;
+      wb = pTsdb->rCache.rwritebatch;
      char  *value = NULL;
      size_t vlen = 0;
      tsdbCacheSerialize(pLastCol, &value, &vlen);
@@ -620,7 +628,7 @@ static SLastCol *tsdbCacheLoadCol(STsdb *pTsdb, SCacheRowsReader *pr, int16_t sl
    }

    if (wb) {
-      rocksMayWrite(pTsdb, false);
+      rocksMayWrite(pTsdb, false, true);
    }

    taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
@@ -671,6 +679,13 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
      pLastCol = &noneCol;
    }

+    taosArraySet(pLastArray, idxKey->idx, pLastCol);
+    // taosArrayRemove(remainCols, i);
+
+    if (!pTmpColArray) {
+      continue;
+    }
+
    SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol));
    *pTmpLastCol = *pLastCol;
    pLastCol = pTmpLastCol;
@@ -687,7 +702,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
    }

    // store result back to rocks cache
-    wb = pTsdb->rCache.writebatch;
+    wb = pTsdb->rCache.rwritebatch;
    char  *value = NULL;
    size_t vlen = 0;
    tsdbCacheSerialize(pLastCol, &value, &vlen);
@@ -696,13 +711,10 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
    size_t    klen = ROCKS_KEY_LEN;
    rocksdb_writebatch_put(wb, (char *)key, klen, value, vlen);
    taosMemoryFree(value);
-
-    taosArraySet(pLastArray, idxKey->idx, pLastCol);
-    // taosArrayRemove(remainCols, i);
  }

  if (wb) {
-    rocksMayWrite(pTsdb, false);
+    rocksMayWrite(pTsdb, false, true);
  }

  taosArrayDestroy(pTmpColArray);
@@ -932,7 +944,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
  size_t *values_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t));
  char  **errs = taosMemoryCalloc(num_keys * 2, sizeof(char *));
  taosThreadMutexLock(&pTsdb->rCache.rMutex);
-  rocksMayWrite(pTsdb, true);
+  rocksMayWrite(pTsdb, true, false);
  rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys * 2, (const char *const *)keys_list,
                    keys_list_sizes, values_list, values_list_sizes, errs);
  for (int i = 0; i < num_keys; ++i) {
@@ -971,7 +983,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
  taosMemoryFree(values_list);
  taosMemoryFree(values_list_sizes);

-  rocksMayWrite(pTsdb, true);
+  rocksMayWrite(pTsdb, true, false);
  taosThreadMutexUnlock(&pTsdb->rCache.rMutex);

 _exit:

--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -3112,6 +3112,10 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
    // load the last data block of current table
    STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter;
    if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pScanInfo->uid, sizeof(pScanInfo->uid))) {
+      // reset the index in last block when handing a new file
+      doCleanupTableScanInfo(pScanInfo);
+      pStatus->mapDataCleaned = true;
+
      bool hasNexTable = moveToNextTable(pUidList, pStatus);
      if (!hasNexTable) {
        return TSDB_CODE_SUCCESS;

--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -2219,6 +2219,13 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) {
  FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) {
    SFunctionNode* pAggFunc = (SFunctionNode*)pFunc;
    if (FUNCTION_TYPE_LAST == pAggFunc->funcType) {
+      SNode* pPar = nodesListGetNode(pAggFunc->pParameterList, 0);
+      if (QUERY_NODE_COLUMN == nodeType(pPar)) {
+        SColumnNode* pCol = (SColumnNode*)pPar;
+        if (pCol->colType == COLUMN_TYPE_TAG) {
+          return false;      
+        }
+      }
      if (hasSelectFunc || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) {
        return false;
      }
@@ -2319,6 +2326,7 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
  if (NULL != cxt.pLastCols) {
    cxt.doAgg = false;
    lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols, true);
+    nodesWalkExprs(pScan->pScanPseudoCols, lastRowScanOptSetColDataType, &cxt);
    lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols, false);
    nodesClearList(cxt.pLastCols);
  }