Merge pull request #5449 from taosdata/hotfix/sangshuduo/TD-3316-testcase-for-taosdemo-limit-offset

Hotfix/sangshuduo/td 3316 testcase for taosdemo limit offset

src/kit/taosdemo/taosdemo.c

@@ -1997,7 +1997,7 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
 
   int childTblCount = (limit < 0)?10000:limit;
   int count = 0;
-  childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
+//  childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
   char* pTblName = childTblName;
   while ((row = taos_fetch_row(res)) != NULL) {
     int32_t* len = taos_fetch_lengths(res);
@@ -3955,7 +3955,7 @@ static int getRowDataFromSample(char*  dataBuf, int maxLen, int64_t timestamp,
   return dataLen;
 }
 
-int generateRowData(char*  dataBuf, int maxLen, int64_t timestamp, SSuperTable* stbInfo) {
+static int generateRowData(char*  dataBuf, int maxLen, int64_t timestamp, SSuperTable* stbInfo) {
   int    dataLen = 0;
   dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp);
   for (int i = 0; i < stbInfo->columnCount; i++) {    
@@ -4343,26 +4343,22 @@ static int execInsert(threadInfo *winfo, char *buffer, int k)
 
   if (superTblInfo) {
     if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) {
-          verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
-          affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE);
+      verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
+      affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE);
     } else {
-          verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
-          int retCode = postProceSql(g_Dbs.host, g_Dbs.port, buffer);
+      verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
+      int retCode = postProceSql(g_Dbs.host, g_Dbs.port, buffer);
 
-          if (0 != retCode) {
-            affectedRows = -1;
-            printf("========restful return fail, threadID[%d]\n", winfo->threadID);
-          } else {
-          affectedRows = k;
-          }
+      if (0 != retCode) {
+        affectedRows = -1;
+        printf("========restful return fail, threadID[%d]\n", winfo->threadID);
+      } else {
+        affectedRows = k;
+      }
     }
   } else {
-        verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
-        affectedRows = queryDbExec(winfo->taos, buffer, 1);
-  }
-
-  if (0 > affectedRows){
-            return affectedRows;
+    verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
+    affectedRows = queryDbExec(winfo->taos, buffer, 1);
   }
 
   return affectedRows;
@@ -4390,6 +4386,12 @@ static int generateDataBuffer(int32_t tableSeq,
   char *pChildTblName;
   int childTblCount;
 
+  pChildTblName = calloc(TSDB_TABLE_NAME_LEN, 1);
+  if (NULL == pChildTblName) {
+    fprintf(stderr, "failed to alloc memory %d\n", TSDB_TABLE_NAME_LEN);
+    return -1;
+  }
+
   if (superTblInfo && (superTblInfo->childTblOffset > 0)) {
       // select tbname from stb limit 1 offset tableSeq
     getChildNameOfSuperTableWithLimitAndOffset(pThreadInfo->taos,
@@ -4397,11 +4399,6 @@ static int generateDataBuffer(int32_t tableSeq,
             &pChildTblName, &childTblCount,
             1, tableSeq);
   } else {
-      pChildTblName = calloc(TSDB_TABLE_NAME_LEN, 1);
-      if (NULL == pChildTblName) {
-          fprintf(stderr, "failed to alloc memory %d\n", TSDB_TABLE_NAME_LEN);
-          return -1;
-      }
       snprintf(pChildTblName, TSDB_TABLE_NAME_LEN, "%s%d",
         superTblInfo?superTblInfo->childTblPrefix:g_args.tb_prefix, tableSeq);
   }
@@ -4421,9 +4418,9 @@ static int generateDataBuffer(int32_t tableSeq,
                     tableSeq % superTblInfo->tagSampleCount);
       }
       if (NULL == tagsValBuf) {
-         fprintf(stderr, "tag buf failed to allocate  memory\n");
-         free(pChildTblName);
-         return -1;
+        free(pChildTblName);
+        fprintf(stderr, "tag buf failed to allocate  memory\n");
+        return -1;
       }
 
       pstr += snprintf(pstr,
@@ -4498,6 +4495,11 @@ static int generateDataBuffer(int32_t tableSeq,
         }
 
         len += retLen;
+
+        if (len >= (superTblInfo->maxSqlLen - 256)) {    // reserve for overwrite
+            k++;
+            break;
+        }
       }
     } else {
       int rand_num = taosRandom() % 100;
@@ -4518,15 +4520,15 @@ static int generateDataBuffer(int32_t tableSeq,
                   lenOfBinary);
       }
 
+      pstr += sprintf(pstr, " %s", data);
       //assert(len + pstr - buffer < BUFFER_SIZE);
-      if (len + pstr - buffer >= g_args.max_sql_len) { // too long
-            break;
+      if (len + pstr - buffer >= (g_args.max_sql_len - 256)) { // too long
+          k++;
+          break;
       }
-
-      pstr += sprintf(pstr, " %s", data);
     }
 
-    verbosePrint("%s() LN%d len=%d k=%d \nbuffer=%s\n", __func__, __LINE__, len, k, buffer);
+    verbosePrint("%s() LN%d len=%d k=%d \nbuffer=%p\n", __func__, __LINE__, len, k, buffer);
 
     k++;
     startFrom ++;
@@ -4535,7 +4537,8 @@ static int generateDataBuffer(int32_t tableSeq,
       break;
   }
 
-  free(pChildTblName);
+  if (pChildTblName)
+      free(pChildTblName);
 
   return k;
 }
@@ -4613,6 +4616,7 @@ static void* syncWrite(void *sarg) {
       if (affectedRows < 0)
         goto free_and_statistics_2;
 
+
       winfo->totalInsertRows += generated;
       winfo->totalAffectedRows += affectedRows;
 
@@ -4712,7 +4716,7 @@ void callBack(void *param, TAOS_RES *res, int code) {
       break;
     }
   }
-  
+
   if (insert_interval) {
     winfo->st = taosGetTimestampUs();
   }

tests/pytest/fulltest.sh

@@ -243,10 +243,12 @@ python3 ./test.py -f query/last_row_cache.py
 python3 ./test.py -f query/last_cache.py
 
 # tools
-python3 test.py -f tools/taosdemoTest.py
 python3 test.py -f tools/taosdumpTest.py
 python3 test.py -f tools/lowaTest.py
+python3 test.py -f tools/taosdemoTest.py
 python3 test.py -f tools/taosdemoTest2.py
+python3 test.py -f tools/taosdemoTestWithoutMetric.py
+python3 test.py -f tools/taosdemoTestLimitOffset.py
 
 # subscribe
 python3 test.py -f subscribe/singlemeter.py
@@ -272,4 +274,4 @@ python3 ./test.py -f update/bug_td2279.py
 python3 ./test.py -f wal/addOldWalTest.py
 
 # account
-python3 ./test.py -f account/account_create.py
\ No newline at end of file
+python3 ./test.py -f account/account_create.py

tests/pytest/pytest_3.sh

@@ -78,10 +78,11 @@ python3 test.py -f query/queryInterval.py
 python3 test.py -f query/queryFillTest.py
 
 # tools
+python3 test.py -f tools/lowaTest.py
 python3 test.py -f tools/taosdemoTest.py
 python3 test.py -f tools/taosdemoTestWithoutMetric.py
+python3 test.py -f tools/taosdemoTestLimitOffset.py
 python3 test.py -f tools/taosdumpTest.py
-python3 test.py -f tools/lowaTest.py
 #python3 test.py -f tools/taosdemoTest2.py
 
 # subscribe

tests/pytest/tools/insert-tblimit-tboffset.json

+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,
+    "thread_count_create_tbl": 4,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "num_of_records_per_req": 100,
+    "max_sql_len": 1024000,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "yes",
+            "replica": 1,
+            "days": 10,
+            "cache": 16,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 365,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb",
+            "child_table_exists":"no",
+            "childtable_count": 100,
+            "childtable_prefix": "stb_",
+            "auto_create_table": "no",
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 1000,
+            "childtable_limit": 33,
+            "childtable_offset": 33,
+            "multi_thread_write_one_tbl": "no",
+            "number_of_tbl_in_one_sql": 0,
+            "rows_per_tbl": 100,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
+        }]
+    }]
+}

tests/pytest/tools/taosdemoTestLimitOffset.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        self.numberOfTables = 10000
+        self.numberOfRecords = 100
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
+
+    def run(self):
+        tdSql.prepare()
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        binPath = buildPath+ "/build/bin/"
+        os.system("%staosdemo -f tools/insert-tblimit-tboffset.json" % binPath)
+
+        tdSql.execute("use db")
+        tdSql.query("select count(tbname) from db.stb")
+        tdSql.checkData(0, 0, 100)
+        tdSql.query("select count(*) from db.stb")
+        tdSql.checkData(0, 0, 33000)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())