Hotfix/sangshuduo/td 5872 taosdemo stmt improve for master (#8009)

* cherry pick from develop branch.

* cherry pick 54813175

* [TD-5872]<fix>: taosdemo stmt csv perf improve.

* rand func back to early impl.

* fix windows/mac compile error.

* cherry pick from develop branch.

* merge with develop branch.

* rename MAX_SAMPLES_ONCE_FROM_FILE to reflect reality.

* split func for stmt interlace.

* fix bug that get build path.

* stmt batch interlace works.

* fix start time issue.

* few cleanup.
Co-authored-by: NShuduo Sang <sdsang@taosdata.com>

src/kit/taosdemo/taosdemo.c

@@ -7746,77 +7746,60 @@ static int execStbBindParamBatch(
 
                 case TSDB_DATA_TYPE_INT:
                     param->buffer_length = sizeof(int32_t);
-                    param->buffer = (stbInfo)?
+                    param->buffer =
                         (void *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray+sizeof(char*)*(c-1))
-                                        + stbInfo->columns[c-1].dataLen * (*pSamplePos)):
-                        (void *)((uintptr_t)*(uintptr_t*)(g_sampleBindBatchArray+sizeof(char*)*(c-1))
-                                    + sizeof(int32_t)*(*pSamplePos));
+                                        + stbInfo->columns[c-1].dataLen * (*pSamplePos));
                     break;
 
                 case TSDB_DATA_TYPE_TINYINT:
                     param->buffer_length = sizeof(int8_t);
-                    param->buffer = (stbInfo)?
+                    param->buffer =
                         (void *)((uintptr_t)*(uintptr_t*)(
                                     stbInfo->sampleBindBatchArray
                                     +sizeof(char*)*(c-1))
-                                + stbInfo->columns[c-1].dataLen*(*pSamplePos)):
-                        (void *)((uintptr_t)*(uintptr_t*)(
-                                    g_sampleBindBatchArray+sizeof(char*)*(c-1))
-                                    + sizeof(int8_t)*(*pSamplePos));
+                                + stbInfo->columns[c-1].dataLen*(*pSamplePos));
                     break;
 
                 case TSDB_DATA_TYPE_SMALLINT:
                     param->buffer_length = sizeof(int16_t);
-                    param->buffer = (stbInfo)?
+                    param->buffer =
                         (void *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray+sizeof(char*)*(c-1))
-                                        + stbInfo->columns[c-1].dataLen * (*pSamplePos)):
-                        (void *)((uintptr_t)*(uintptr_t*)(g_sampleBindBatchArray+sizeof(char*)*(c-1))
-                                    + sizeof(int16_t)*(*pSamplePos));
+                                        + stbInfo->columns[c-1].dataLen * (*pSamplePos));
                     break;
 
                 case TSDB_DATA_TYPE_BIGINT:
                     param->buffer_length = sizeof(int64_t);
-                    param->buffer = (stbInfo)?
+                    param->buffer =
                         (void *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray+sizeof(char*)*(c-1))
-                                        + stbInfo->columns[c-1].dataLen * (*pSamplePos)):
-                        (void *)((uintptr_t)*(uintptr_t*)(g_sampleBindBatchArray+sizeof(char*)*(c-1))
-                                    + sizeof(int64_t)*(*pSamplePos));
+                                        + stbInfo->columns[c-1].dataLen * (*pSamplePos));
                     break;
 
                 case TSDB_DATA_TYPE_BOOL:
                     param->buffer_length = sizeof(int8_t);
-                    param->buffer = (stbInfo)?
+                    param->buffer =
                         (void *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray+sizeof(char*)*(c-1))
-                                        + stbInfo->columns[c-1].dataLen * (*pSamplePos)):
-                        (void *)((uintptr_t)*(uintptr_t*)(g_sampleBindBatchArray+sizeof(char*)*(c-1))
-                                    + sizeof(int8_t)*(*pSamplePos));
+                                        + stbInfo->columns[c-1].dataLen * (*pSamplePos));
                     break;
 
                 case TSDB_DATA_TYPE_FLOAT:
                     param->buffer_length = sizeof(float);
-                    param->buffer = (stbInfo)?
+                    param->buffer =
                         (void *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray+sizeof(char*)*(c-1))
-                                        + stbInfo->columns[c-1].dataLen * (*pSamplePos)):
-                        (void *)((uintptr_t)*(uintptr_t*)(g_sampleBindBatchArray+sizeof(char*)*(c-1))
-                                    + sizeof(float)*(*pSamplePos));
+                                        + stbInfo->columns[c-1].dataLen * (*pSamplePos));
                     break;
 
                 case TSDB_DATA_TYPE_DOUBLE:
                     param->buffer_length = sizeof(double);
-                    param->buffer = (stbInfo)?
+                    param->buffer =
                         (void *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray+sizeof(char*)*(c-1))
-                                        + stbInfo->columns[c-1].dataLen * (*pSamplePos)):
-                        (void *)((uintptr_t)*(uintptr_t*)(g_sampleBindBatchArray+sizeof(char*)*(c-1))
-                                    + sizeof(double)*(*pSamplePos));
+                                        + stbInfo->columns[c-1].dataLen * (*pSamplePos));
                     break;
 
                 case TSDB_DATA_TYPE_TIMESTAMP:
                     param->buffer_length = sizeof(int64_t);
-                    param->buffer = (stbInfo)?
+                    param->buffer =
                         (void *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray+sizeof(char*)*(c-1))
-                                        + stbInfo->columns[c-1].dataLen * (*pSamplePos)):
-                        (void *)((uintptr_t)*(uintptr_t*)(g_sampleBindBatchArray+sizeof(char*)*(c-1))
-                                    + sizeof(int64_t)*(*pSamplePos));
+                                        + stbInfo->columns[c-1].dataLen * (*pSamplePos));
                     break;
 
                 default:
@@ -7837,7 +7820,7 @@ static int execStbBindParamBatch(
             if (param->buffer_type == TSDB_DATA_TYPE_NCHAR) {
                 param->length[b] = strlen(
                      (char *)param->buffer + b *
-                         ((stbInfo)?stbInfo->columns[c].dataLen:g_args.binwidth)
+                         stbInfo->columns[c].dataLen
                         );
             } else {
                 param->length[b] = param->buffer_length;
@@ -10072,15 +10055,18 @@ static void startMultiThreadInsertData(int threads, char* db_name,
         }
     }
 
-    fprintf(stderr, "insert delay, avg: %10.2fms, max: %10.2fms, min: %10.2fms\n\n",
-            (double)avgDelay/1000.0,
-            (double)maxDelay/1000.0,
-            (double)minDelay/1000.0);
-    if (g_fpOfInsertResult) {
-        fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %10.2fms, min: %10.2fms\n\n",
-            (double)avgDelay/1000.0,
-            (double)maxDelay/1000.0,
-            (double)minDelay/1000.0);
+    if (minDelay != UINT64_MAX) {
+        fprintf(stderr, "insert delay, avg: %10.2fms, max: %10.2fms, min: %10.2fms\n\n",
+                (double)avgDelay/1000.0,
+                (double)maxDelay/1000.0,
+                (double)minDelay/1000.0);
+
+        if (g_fpOfInsertResult) {
+            fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %10.2fms, min: %10.2fms\n\n",
+                    (double)avgDelay/1000.0,
+                    (double)maxDelay/1000.0,
+                    (double)minDelay/1000.0);
+        }
     }
 
     //taos_close(taos);

tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdumpTestNanoSupport.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        self.ts = 1625068800000000000   # this is timestamp  "2021-07-01 00:00:00"
+        self.numberOfTables = 10
+        self.numberOfRecords = 100
+
+    def checkCommunity(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+        if ("community" in selfPath):
+            return False
+        else:
+            return True
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+    
+
+
+    def createdb(self, precision="ns"):
+        tb_nums = self.numberOfTables
+        per_tb_rows = self.numberOfRecords
+
+        def build_db(precision, start_time):
+            tdSql.execute("drop database if exists timedb1")
+            tdSql.execute(
+                "create database timedb1 days 10 keep 365 blocks 8 precision "+"\""+precision+"\"")
+
+            tdSql.execute("use timedb1")
+            tdSql.execute(
+                "create stable st(ts timestamp, c1 int, c2 nchar(10),c3 timestamp) tags(t1 int, t2 binary(10))")
+            for tb in range(tb_nums):
+                tbname = "t"+str(tb)
+                tdSql.execute("create table " + tbname +
+                              " using st tags(1, 'beijing')")
+                sql = "insert into " + tbname + " values"
+                currts = start_time
+                if precision == "ns":
+                    ts_seed = 1000000000
+                elif precision == "us":
+                    ts_seed = 1000000
+                else:
+                    ts_seed = 1000
+
+                for i in range(per_tb_rows):
+                    sql += "(%d, %d, 'nchar%d',%d)" % (currts + i*ts_seed, i %
+                                                       100, i % 100, currts + i*100)  # currts +1000ms (1000000000ns)
+                tdSql.execute(sql)
+
+        if precision == "ns":
+            start_time = 1625068800000000000
+            build_db(precision, start_time)
+
+        elif precision == "us":
+            start_time = 1625068800000000
+            build_db(precision, start_time)
+
+        elif precision == "ms":
+            start_time = 1625068800000
+            build_db(precision, start_time)
+
+        else:
+            print("other time precision not valid , please check! ")
+    
+
+    def run(self):
+
+        # clear envs
+        os.system("rm -rf ./taosdumptest/")
+        tdSql.execute("drop database if exists dumptmp1")
+        tdSql.execute("drop database if exists dumptmp2")
+        tdSql.execute("drop database if exists dumptmp3")
+
+        if not os.path.exists("./taosdumptest/tmp1"):
+            os.makedirs("./taosdumptest/dumptmp1")
+        else:
+            print("path exist!")
+
+        if not os.path.exists("./taosdumptest/dumptmp2"):
+            os.makedirs("./taosdumptest/dumptmp2")
+
+        if not os.path.exists("./taosdumptest/dumptmp3"):
+            os.makedirs("./taosdumptest/dumptmp3")
+
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosdump not found!")
+        else:
+            tdLog.info("taosdump found in %s" % buildPath)
+        binPath = buildPath + "/build/bin/"
+
+        # create nano second database
+
+        self.createdb(precision="ns")
+
+        # dump all data
+
+        os.system(
+            "%staosdump --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath)
+
+        # dump part data with -S  -E
+        os.system(
+            '%staosdump --databases timedb1 -S 1625068810000000000 -E 1625068860000000000  -C ns  -o ./taosdumptest/dumptmp2 ' %
+            binPath)
+        os.system(
+            '%staosdump --databases timedb1 -S 1625068810000000000  -o ./taosdumptest/dumptmp3  ' %
+            binPath)
+
+        # replace strings to dump in databases
+        os.system(
+            "sed -i \"s/timedb1/dumptmp1/g\" `grep timedb1 -rl ./taosdumptest/dumptmp1`")
+        os.system(
+            "sed -i \"s/timedb1/dumptmp2/g\" `grep timedb1 -rl ./taosdumptest/dumptmp2`")
+        os.system(
+            "sed -i \"s/timedb1/dumptmp3/g\" `grep timedb1 -rl ./taosdumptest/dumptmp3`")
+
+        os.system( "%staosdump -i ./taosdumptest/dumptmp1" %binPath)
+        os.system( "%staosdump -i ./taosdumptest/dumptmp2" %binPath)
+        os.system( "%staosdump -i ./taosdumptest/dumptmp3" %binPath)
+
+        # dump data and check for taosdump
+        tdSql.query("select count(*) from dumptmp1.st")
+        tdSql.checkData(0,0,1000)
+
+        tdSql.query("select count(*) from dumptmp2.st")
+        tdSql.checkData(0,0,510)
+
+        tdSql.query("select count(*) from dumptmp3.st")
+        tdSql.checkData(0,0,900)
+
+        # check data
+        origin_res = tdSql.getResult("select * from timedb1.st")
+        dump_res = tdSql.getResult("select * from dumptmp1.st")
+        if origin_res == dump_res:
+            tdLog.info("test nano second : dump check data pass for all data!" )
+        else:
+            tdLog.info("test nano second : dump check data failed for all data!" )
+        
+        origin_res = tdSql.getResult("select * from timedb1.st where ts >=1625068810000000000 and ts <= 1625068860000000000")
+        dump_res = tdSql.getResult("select * from dumptmp2.st")
+        if origin_res == dump_res:
+            tdLog.info(" test nano second : dump check data pass for data! " )
+        else:
+            tdLog.info(" test nano second : dump check data failed for data !" )
+        
+        origin_res = tdSql.getResult("select * from timedb1.st where ts >=1625068810000000000 ")
+        dump_res = tdSql.getResult("select * from dumptmp3.st")
+        if origin_res == dump_res:
+            tdLog.info(" test nano second : dump check data pass for data! " )
+        else:
+            tdLog.info(" test nano second : dump check data failed for data !" )
+        
+
+        # us second support test case
+
+        os.system("rm -rf ./taosdumptest/")
+        tdSql.execute("drop database if exists dumptmp1")
+        tdSql.execute("drop database if exists dumptmp2")
+        tdSql.execute("drop database if exists dumptmp3")
+
+        if not os.path.exists("./taosdumptest/tmp1"):
+            os.makedirs("./taosdumptest/dumptmp1")
+        else:
+            print("path exits!")
+
+        if not os.path.exists("./taosdumptest/dumptmp2"):
+            os.makedirs("./taosdumptest/dumptmp2")
+
+        if not os.path.exists("./taosdumptest/dumptmp3"):
+            os.makedirs("./taosdumptest/dumptmp3")
+
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosdump not found!")
+        else:
+            tdLog.info("taosdump found in %s" % buildPath)
+        binPath = buildPath + "/build/bin/"
+
+        self.createdb(precision="us")
+
+        os.system(
+            "%staosdump --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath)
+
+        os.system(
+            '%staosdump --databases timedb1 -S 1625068810000000 -E 1625068860000000  -C us  -o ./taosdumptest/dumptmp2 ' %
+            binPath)
+        os.system(
+            '%staosdump --databases timedb1 -S 1625068810000000  -o ./taosdumptest/dumptmp3  ' %
+            binPath)
+
+        os.system(
+            "sed -i \"s/timedb1/dumptmp1/g\" `grep timedb1 -rl ./taosdumptest/dumptmp1`")
+        os.system(
+            "sed -i \"s/timedb1/dumptmp2/g\" `grep timedb1 -rl ./taosdumptest/dumptmp2`")
+        os.system(
+            "sed -i \"s/timedb1/dumptmp3/g\" `grep timedb1 -rl ./taosdumptest/dumptmp3`")
+
+        os.system( "%staosdump -i ./taosdumptest/dumptmp1" %binPath)
+        os.system( "%staosdump -i ./taosdumptest/dumptmp2" %binPath)
+        os.system( "%staosdump -i ./taosdumptest/dumptmp3" %binPath)
+
+ 
+        tdSql.query("select count(*) from dumptmp1.st")
+        tdSql.checkData(0,0,1000)
+
+        tdSql.query("select count(*) from dumptmp2.st")
+        tdSql.checkData(0,0,510)
+
+        tdSql.query("select count(*) from dumptmp3.st")
+        tdSql.checkData(0,0,900)
+
+    
+        origin_res = tdSql.getResult("select * from timedb1.st")
+        dump_res = tdSql.getResult("select * from dumptmp1.st")
+        if origin_res == dump_res:
+            tdLog.info("test us second : dump check data pass for all data!" )
+        else:
+            tdLog.info("test us second : dump check data failed for all data!" )
+        
+        origin_res = tdSql.getResult("select * from timedb1.st where ts >=1625068810000000 and ts <= 1625068860000000")
+        dump_res = tdSql.getResult("select * from dumptmp2.st")
+        if origin_res == dump_res:
+            tdLog.info(" test us second : dump check data pass for data! " )
+        else:
+            tdLog.info(" test us second : dump check data failed for data!" )
+        
+        origin_res = tdSql.getResult("select * from timedb1.st where ts >=1625068810000000 ")
+        dump_res = tdSql.getResult("select * from dumptmp3.st")
+        if origin_res == dump_res:
+            tdLog.info(" test us second : dump check data pass for data! " )
+        else:
+            tdLog.info(" test us second : dump check data failed for data! " )
+
+        
+        # ms second support test case
+
+        os.system("rm -rf ./taosdumptest/")
+        tdSql.execute("drop database if exists dumptmp1")
+        tdSql.execute("drop database if exists dumptmp2")
+        tdSql.execute("drop database if exists dumptmp3")
+
+        if not os.path.exists("./taosdumptest/tmp1"):
+            os.makedirs("./taosdumptest/dumptmp1")
+        else:
+            print("path exits!")
+
+        if not os.path.exists("./taosdumptest/dumptmp2"):
+            os.makedirs("./taosdumptest/dumptmp2")
+
+        if not os.path.exists("./taosdumptest/dumptmp3"):
+            os.makedirs("./taosdumptest/dumptmp3")
+
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosdump not found!")
+        else:
+            tdLog.info("taosdump found in %s" % buildPath)
+        binPath = buildPath + "/build/bin/"
+
+        self.createdb(precision="ms")
+
+        os.system(
+            "%staosdump --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath)
+
+        os.system(
+            '%staosdump --databases timedb1 -S 1625068810000 -E 1625068860000  -C ms  -o ./taosdumptest/dumptmp2 ' %
+            binPath)
+        os.system(
+            '%staosdump --databases timedb1 -S 1625068810000  -o ./taosdumptest/dumptmp3  ' %
+            binPath)
+
+        os.system(
+            "sed -i \"s/timedb1/dumptmp1/g\" `grep timedb1 -rl ./taosdumptest/dumptmp1`")
+        os.system(
+            "sed -i \"s/timedb1/dumptmp2/g\" `grep timedb1 -rl ./taosdumptest/dumptmp2`")
+        os.system(
+            "sed -i \"s/timedb1/dumptmp3/g\" `grep timedb1 -rl ./taosdumptest/dumptmp3`")
+
+        os.system( "%staosdump -i ./taosdumptest/dumptmp1" %binPath)
+        os.system( "%staosdump -i ./taosdumptest/dumptmp2" %binPath)
+        os.system( "%staosdump -i ./taosdumptest/dumptmp3" %binPath)
+
+ 
+        tdSql.query("select count(*) from dumptmp1.st")
+        tdSql.checkData(0,0,1000)
+
+        tdSql.query("select count(*) from dumptmp2.st")
+        tdSql.checkData(0,0,510)
+
+        tdSql.query("select count(*) from dumptmp3.st")
+        tdSql.checkData(0,0,900)
+
+    
+        origin_res = tdSql.getResult("select * from timedb1.st")
+        dump_res = tdSql.getResult("select * from dumptmp1.st")
+        if origin_res == dump_res:
+            tdLog.info("test ms second : dump check data pass for all data!" )
+        else:
+            tdLog.info("test ms second : dump check data failed for all data!" )
+        
+        origin_res = tdSql.getResult("select * from timedb1.st where ts >=1625068810000 and ts <= 1625068860000")
+        dump_res = tdSql.getResult("select * from dumptmp2.st")
+        if origin_res == dump_res:
+            tdLog.info(" test ms second : dump check data pass for data! " )
+        else:
+            tdLog.info(" test ms second : dump check data failed for data!" )
+        
+        origin_res = tdSql.getResult("select * from timedb1.st where ts >=1625068810000 ")
+        dump_res = tdSql.getResult("select * from dumptmp3.st")
+        if origin_res == dump_res:
+            tdLog.info(" test ms second : dump check data pass for data! " )
+        else:
+            tdLog.info(" test ms second : dump check data failed for data! " )
+
+        
+        os.system("rm -rf ./taosdumptest/")
+        os.system("rm -rf ./dump_result.txt")
+        os.system("rm -rf *.py.sql")
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())