Merge branch 'develop' into enhance/TS-385

cmake/env.inc

@@ -2,7 +2,12 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
 PROJECT(TDengine)
 
 SET(CMAKE_C_STANDARD 11)
-SET(CMAKE_VERBOSE_MAKEFILE ON)
+
+IF (TD_BUILD_VERBOSE)
+  SET(CMAKE_VERBOSE_MAKEFILE ON)
+ELSE ()
+  SET(CMAKE_VERBOSE_MAKEFILE OFF)
+ENDIF ()
 
 #set output directory
 SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)

cmake/input.inc

@@ -101,6 +101,12 @@ IF (${MEMORY_SANITIZER} MATCHES "true")
   SET(TD_MEMORY_SANITIZER TRUE)
 ENDIF ()
 
+SET(TD_BUILD_VERBOSE FALSE)
+IF (${VERBOSE} MATCHES "true")
+  SET(CMAKE_VERBOSE_MAKEFILE ON)
+  SET(TD_BUILD_VERBOSE TRUE)
+ENDIF ()
+
 IF (${TSZ_ENABLED} MATCHES "true")
   # define add 
   MESSAGE(STATUS "build with TSZ enabled")

documentation20/cn/02.getting-started/docs.md

@@ -175,8 +175,9 @@ taos> select avg(current), max(voltage), min(phase) from test.meters where group
 ```mysql
 taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s);
 ```
+## <a class="anchor" id="taosdemo"></a> taosdemo 详细功能列表
 
-**Note:** taosdemo 命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help` 详细列出。您可以设置不同参数进行体验。
+taosdemo 命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help` 详细列出。您可以设置不同参数进行体验。
 taosdemo 详细使用方法请参照 [如何使用taosdemo对TDengine进行性能测试？](https://www.taosdata.com/cn/documentation/getting-started/taosdemo )。
 
 ## 客户端和报警模块

documentation20/en/02.getting-started/docs.md

@@ -180,7 +180,9 @@ taos> select avg(f1), max(f2), min(f3) from test.meters where areaid=10;
 taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s);
 ```
 
-**Note**: you can run command `taosdemo` with many options, like number of tables, rows of records and so on. To know more about these options, you can execute `taosdemo --help` and then take a try using different options.
+## <a class="anchor" id="taosdemo"></a> Using taosdemo in detail
+
+you can run command `taosdemo` with many options, like number of tables, rows of records and so on. To know more about these options, you can execute `taosdemo --help` and then take a try using different options.
 Please refer to [How to use taosdemo to test the performance of TDengine](https://www.taosdata.com/en/documentation/getting-started/taosdemo) for detail.
 
 ## Client and Alarm Module

src/client/src/tscParseOpenTSDB.c

@@ -42,7 +42,7 @@ static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index,
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
   if (isdigit(*cur)) {
-    tscError("OTD:0x%"PRIx64" Metric cannnot start with digit", info->id);
+    tscError("OTD:0x%"PRIx64" Metric cannot start with digit", info->id);
     tfree(pSml->stableName);
     return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
   }
@@ -188,7 +188,7 @@ static int32_t parseTelnetTagKey(TAOS_SML_KV *pKV, const char **index, SHashObj
 
   //key field cannot start with digit
   if (isdigit(*cur)) {
-    tscError("OTD:0x%"PRIx64" Tag key cannnot start with digit", info->id);
+    tscError("OTD:0x%"PRIx64" Tag key cannot start with digit", info->id);
     return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
   }
   while (*cur != '\0') {
@@ -476,7 +476,7 @@ static int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlL
   }
 
   if (isdigit(metric->valuestring[0])) {
-    tscError("OTD:0x%"PRIx64" Metric cannnot start with digit in JSON", info->id);
+    tscError("OTD:0x%"PRIx64" Metric cannot start with digit in JSON", info->id);
     tfree(pSml->stableName);
     return TSDB_CODE_TSC_INVALID_JSON;
   }

src/client/src/tscSub.c

@@ -265,12 +265,14 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
 
   SSqlCmd* pCmd = &pSql->cmd;
 
-  TSDB_QUERY_CLEAR_TYPE(tscGetQueryInfo(pCmd)->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
+  SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
+
+  TSDB_QUERY_CLEAR_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
 
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd,  0);
   if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
     STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
-    SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = 0};
+    SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = pQueryInfo->window.skey};
     SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
     if (p == NULL) {
       taosArrayClear(pSub->progress);
@@ -288,7 +290,6 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
   }
   size_t numOfTables = taosArrayGetSize(tables);
 
-  SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
   SArray* progress = taosArrayInit(numOfTables, sizeof(SSubscriptionProgress));
   for( size_t i = 0; i < numOfTables; i++ ) {
     STidTags* tt = taosArrayGet( tables, i );

src/kit/taosdemo/taosdemo.c

@@ -303,6 +303,7 @@ typedef struct SSuperTable_S {
     uint64_t     lenOfTagOfOneRow;
 
     char*        sampleDataBuf;
+    bool         useSampleTs;
 
     uint32_t     tagSource;    // 0: rand, 1: tag sample
     char*        tagDataBuf;
@@ -1687,10 +1688,10 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) {
                         arguments->data_type[index] = TSDB_DATA_TYPE_DOUBLE;
                     } else if (0 == strcasecmp(token, "TINYINT")) {
                         arguments->data_type[index] = TSDB_DATA_TYPE_TINYINT;
-                    } else if (1 == regexMatch(token, "^BINARY(\\([1-9][0-9]*\\))?$", REG_ICASE | 
+                    } else if (1 == regexMatch(token, "^BINARY(\\([1-9][0-9]*\\))?$", REG_ICASE |
                     REG_EXTENDED)) {
                         arguments->data_type[index] = TSDB_DATA_TYPE_BINARY;
-                    } else if (1 == regexMatch(token, "^NCHAR(\\([1-9][0-9]*\\))?$", REG_ICASE | 
+                    } else if (1 == regexMatch(token, "^NCHAR(\\([1-9][0-9]*\\))?$", REG_ICASE |
                     REG_EXTENDED)) {
                         arguments->data_type[index] = TSDB_DATA_TYPE_NCHAR;
                     } else if (0 == strcasecmp(token, "BOOL")) {
@@ -2761,6 +2762,8 @@ static int printfInsertMeta() {
                         g_Dbs.db[i].superTbls[j].sampleFormat);
                 printf("      sampleFile:        \033[33m%s\033[0m\n",
                         g_Dbs.db[i].superTbls[j].sampleFile);
+                printf("      useSampleTs:       \033[33m%s\033[0m\n",
+                        g_Dbs.db[i].superTbls[j].useSampleTs ? "yes (warning: disorderRange/disorderRatio is disabled)" : "no");
                 printf("      tagsFile:          \033[33m%s\033[0m\n",
                         g_Dbs.db[i].superTbls[j].tagsFile);
                 printf("      columnCount:       \033[33m%d\033[0m\n        ",
@@ -2805,8 +2808,6 @@ static int printfInsertMeta() {
             printf("  insertRows:        \033[33m%"PRId64"\033[0m\n",
                         g_args.insertRows);
         }
-        
-        
         printf("\n");
     }
 
@@ -3978,21 +3979,21 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName,
                     (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
                     fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
 
-            
+
             if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
-                        "INT", strlen("INT")) && 
+                        "INT", strlen("INT")) &&
                         strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) {
                 superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_INT;
             } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
-                        "TINYINT", strlen("TINYINT")) && 
+                        "TINYINT", strlen("TINYINT")) &&
                         strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) {
                 superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_TINYINT;
             } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
-                        "SMALLINT", strlen("SMALLINT")) && 
+                        "SMALLINT", strlen("SMALLINT")) &&
                         strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) {
                 superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_SMALLINT;
             } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
-                        "BIGINT", strlen("BIGINT")) && 
+                        "BIGINT", strlen("BIGINT")) &&
                         strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) {
                 superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_BIGINT;
             } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
@@ -4046,7 +4047,7 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName,
                     (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
                     min(NOTE_BUFF_LEN,
                         fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes) + 1);
-            
+
             if (strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) {
                 tstrncpy(superTbls->columns[columnIndex].dataType,
                     (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
@@ -4795,6 +4796,23 @@ static int readTagFromCsvFileToMem(SSuperTable  * stbInfo) {
     return 0;
 }
 
+static void getAndSetRowsFromCsvFile(SSuperTable *stbInfo) {
+    FILE *fp = fopen(stbInfo->sampleFile, "r");
+    int line_count = 0;
+    if (fp == NULL) {
+        errorPrint("Failed to open sample file: %s, reason:%s\n",
+                stbInfo->sampleFile, strerror(errno));
+        exit(EXIT_FAILURE);
+    }
+    char *buf = calloc(1, stbInfo->maxSqlLen);
+    while (fgets(buf, stbInfo->maxSqlLen, fp)) {
+        line_count++;
+    }
+    fclose(fp);
+    tmfree(buf);
+    stbInfo->insertRows = line_count;
+}
+
 /*
    Read 10000 lines at most. If more than 10000 lines, continue to read after using
    */
@@ -5688,6 +5706,23 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
                 goto PARSE_OVER;
             }
 
+            cJSON *useSampleTs = cJSON_GetObjectItem(stbInfo, "use_sample_ts");
+            if (useSampleTs && useSampleTs->type == cJSON_String
+                    && useSampleTs->valuestring != NULL) {
+                if (0 == strncasecmp(useSampleTs->valuestring, "yes", 3)) {
+                    g_Dbs.db[i].superTbls[j].useSampleTs = true;
+                } else if (0 == strncasecmp(useSampleTs->valuestring, "no", 2)){
+                    g_Dbs.db[i].superTbls[j].useSampleTs = false;
+                } else {
+                    g_Dbs.db[i].superTbls[j].useSampleTs = false;
+                }
+            } else if (!useSampleTs) {
+                g_Dbs.db[i].superTbls[j].useSampleTs = false;
+            } else {
+                errorPrint("%s", "failed to read json, use_sample_ts not found\n");
+                goto PARSE_OVER;
+            }
+
             cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
             if ((tagsFile && tagsFile->type == cJSON_String)
                     && (tagsFile->valuestring != NULL)) {
@@ -6448,13 +6483,20 @@ static int getRowDataFromSample(
     }
 
     int    dataLen = 0;
-
-    dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
+    if(stbInfo->useSampleTs) {
+        dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
+            "(%s",
+            stbInfo->sampleDataBuf
+            + stbInfo->lenOfOneRow * (*sampleUsePos));
+    } else {
+        dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
             "(%" PRId64 ", ", timestamp);
-    dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
+        dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
             "%s",
             stbInfo->sampleDataBuf
             + stbInfo->lenOfOneRow * (*sampleUsePos));
+    }
+    
     dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
 
     (*sampleUsePos)++;
@@ -6526,7 +6568,7 @@ static int64_t generateStbRowData(
                     tmpLen = strlen(tmp);
                     tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, BIGINT_BUFF_LEN));
                     break;
-                
+
                 case TSDB_DATA_TYPE_UBIGINT:
                     tmp = rand_ubigint_str();
                     tmpLen = strlen(tmp);
@@ -6887,6 +6929,9 @@ static int prepareSampleForStb(SSuperTable *stbInfo) {
 
     int ret;
     if (0 == strncasecmp(stbInfo->dataSource, "sample", strlen("sample"))) {
+        if(stbInfo->useSampleTs) {
+            getAndSetRowsFromCsvFile(stbInfo);
+        }
         ret = generateSampleFromCsvForStb(stbInfo);
     } else {
         ret = generateSampleFromRandForStb(stbInfo);
@@ -7413,7 +7458,7 @@ static int32_t prepareStmtBindArrayByType(
             bind->length = &bind->buffer_length;
             bind->is_null = NULL;
             break;
-        
+
         case TSDB_DATA_TYPE_UINT:
             bind_uint = malloc(sizeof(uint32_t));
             assert(bind_uint);

tests/CMakeLists.txt

@@ -7,7 +7,6 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
 PROJECT(TDengine)
 
 SET(CMAKE_C_STANDARD 11)
-SET(CMAKE_VERBOSE_MAKEFILE ON)
 
 ADD_SUBDIRECTORY(examples/c)
 ADD_SUBDIRECTORY(tsim)