[TD-3192] <feature>: support stb limit and offset. create tables works.

src/kit/taosdemo/taosdemo.c

@@ -1797,8 +1797,7 @@ int postProceSql(char* host, uint16_t port, char* sqlstr)
     return 0;
 }
 
-
-char* getTagValueFromTagSample(        SSuperTable* stbInfo, int tagUsePos) {
+static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
   char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
   if (NULL == dataBuf) {
     printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
@@ -1806,12 +1805,13 @@ char* getTagValueFromTagSample(        SSuperTable* stbInfo, int tagUsePos) {
   }
 
   int    dataLen = 0;
-  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);
+  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
+          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);
 
   return dataBuf;
 }
 
-char* generateTagVaulesForStb(SSuperTable* stbInfo) {
+static char* generateTagVaulesForStb(SSuperTable* stbInfo) {
   char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
   if (NULL == dataBuf) {
     printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
@@ -1821,9 +1821,11 @@ char* generateTagVaulesForStb(SSuperTable* stbInfo) {
   int    dataLen = 0;
   dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
   for (int i = 0; i < stbInfo->tagCount; i++) {
-    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", 6)) || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", 5))) {
+    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
+            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
       if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
-        printf("binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN);
+        printf("binary or nchar length overflow, max size:%u\n",
+                (uint32_t)TSDB_MAX_BINARY_LEN);
         tmfree(dataBuf);
         return NULL;
       }
@@ -1835,30 +1837,48 @@ char* generateTagVaulesForStb(SSuperTable* stbInfo) {
         return NULL;
       }
       rand_string(buf, stbInfo->tags[i].dataLen);
-      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "\'%s\', ", buf);
+      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
+              "\'%s\', ", buf);
       tmfree(buf);
-    } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "int", 3)) {
-      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_int());
-    } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bigint", 6)) {
-      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64", ", rand_bigint());
-    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType, "float", 5)) {
-      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%f, ", rand_float());
-    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType, "double", 6)) {
-      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%f, ", rand_double());
-    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType, "smallint", 8)) {
-      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_smallint());
-    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType, "tinyint", 7)) {
-      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_tinyint());
-    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bool", 4)) {
-      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_bool());
-    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType, "timestamp", 4)) {
-      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64", ", rand_bigint());
+    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
+                "int", strlen("int"))) {
+      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
+              "%d, ", rand_int());
+    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
+                "bigint", strlen("bigint"))) {
+      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
+              "%"PRId64", ", rand_bigint());
+    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
+                "float", strlen("float"))) {
+      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
+              "%f, ", rand_float());
+    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
+                "double", strlen("double"))) {
+      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
+              "%f, ", rand_double());
+    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
+                "smallint", strlen("smallint"))) {
+      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
+              "%d, ", rand_smallint());
+    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
+                "tinyint", strlen("tinyint"))) {
+      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
+              "%d, ", rand_tinyint());
+    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
+                "bool", strlen("bool"))) {
+      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
+              "%d, ", rand_bool());
+    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
+                "timestamp", strlen("timestamp"))) {
+      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
+              "%"PRId64", ", rand_bigint());
     }  else {
       printf("No support data type: %s\n", stbInfo->tags[i].dataType);
       tmfree(dataBuf);
       return NULL;
     }
   }
+
   dataLen -= 2;
   dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");  
   return dataBuf;
@@ -2423,7 +2443,7 @@ static void* createTable(void *sarg)
 }
 
 int startMultiThreadCreateChildTable(
-        char* cols, int threads, int ntables,
+        char* cols, int threads, int startFrom, int ntables,
         char* db_name, SSuperTable* superTblInfo) {
   pthread_t *pids = malloc(threads * sizeof(pthread_t));
   threadInfo *infos = malloc(threads * sizeof(threadInfo));
@@ -2446,7 +2466,7 @@ int startMultiThreadCreateChildTable(
   int b = 0;
   b = ntables % threads;
  
-  int last = 0;
+  int last = startFrom;
   for (int i = 0; i < threads; i++) {
     threadInfo *t_info = infos + i;
     t_info->threadID = i;
@@ -2505,12 +2525,23 @@ static void createChildTables() {
 
         verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
                 g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
+        int startFrom;
+        if (g_Dbs.db[i].superTbls[j].childTblOffset) {
+            startFrom = g_Dbs.db[i].superTbls[j].childTblOffset;
+            g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblLimit;
+        } else {
+            startFrom = 0;
+            g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;
+        }
+
+        verbosePrint("%s() LN%d: create %d child tables from %d\n", __func__, __LINE__,
+              g_totalChildTables, startFrom);  
         startMultiThreadCreateChildTable(
               g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
               g_Dbs.threadCountByCreateTbl,
-              g_Dbs.db[i].superTbls[j].childTblCount,
+              startFrom,
+              g_totalChildTables,
               g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
-        g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;
       }
     } else {
         // normal table
@@ -2538,6 +2569,7 @@ static void createChildTables() {
         startMultiThreadCreateChildTable(
               tblColsBuf,
               g_Dbs.threadCountByCreateTbl,
+              0,
               g_args.num_of_tables,
               g_Dbs.db[i].dbName,
               NULL);
@@ -2545,30 +2577,6 @@ static void createChildTables() {
   }
 }
 
-/*
-static int taosGetLineNum(const char *fileName)
-{
-  int lineNum = 0;
-  char cmd[1024] = { 0 };
-  char buf[1024] = { 0 };
-  sprintf(cmd, "wc -l %s", fileName);
-
-  FILE *fp = popen(cmd, "r");
-  if (fp == NULL) {
-    fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
-    return lineNum;
-  }
-
-  if (fgets(buf, sizeof(buf), fp)) {
-    int index = strchr((const char*)buf, ' ') - buf;
-    buf[index] = '\0';
-    lineNum = atoi(buf);
-  }
-  pclose(fp);
-  return lineNum;
-}
-*/
-
 /*
   Read 10000 lines at most. If more than 10000 lines, continue to read after using
 */
@@ -4010,7 +4018,7 @@ static void syncWriteForNumberOfTblInOneSql(
   int64_t et = 0xffffffff;
   for (int i = 0; i < superTblInfo->insertRows;) {
     int32_t  tbl_id = 0;
-    for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; ) {
+    for (int tableID = winfo->start_table_id; tableID <= winfo->end_table_id; ) {
       int64_t start_time = 0;
       int inserted = i;
 
@@ -4019,12 +4027,12 @@ static void syncWriteForNumberOfTblInOneSql(
         memset(buffer, 0, superTblInfo->maxSqlLen);
         char *pstr = buffer;
 
-        int32_t  end_tbl_id = tID + numberOfTblInOneSql;
+        int32_t  end_tbl_id = tableID + numberOfTblInOneSql;
         if (end_tbl_id > winfo->end_table_id) {
           end_tbl_id = winfo->end_table_id+1;
         }
 
-        for (tbl_id = tID; tbl_id < end_tbl_id; tbl_id++) {  
+        for (tbl_id = tableID; tbl_id < end_tbl_id; tbl_id++) {  
           sampleUsePos = samplePos;
           if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
             char* tagsValBuf = NULL;
@@ -4132,7 +4140,7 @@ static void syncWriteForNumberOfTblInOneSql(
 
             if (inserted >= superTblInfo->insertRows || 
                     (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128)) {
-              tID = tbl_id + 1;
+              tableID = tbl_id + 1;
               printf("config rowsPerTbl and numberOfTblInOneSql not match with max_sql_lenth, please reconfig![lenOfOneRow:%d]\n", 
                       superTblInfo->lenOfOneRow);
               goto send_to_server;
@@ -4140,7 +4148,7 @@ static void syncWriteForNumberOfTblInOneSql(
           }
         }
 
-        tID = tbl_id;
+        tableID = tbl_id;
         inserted += superTblInfo->rowsPerTbl;
 
 send_to_server:
@@ -4209,7 +4217,7 @@ send_to_server:
         break;
       }
 
-      if (tID > winfo->end_table_id) {
+      if (tableID > winfo->end_table_id) {
         if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) {
           samplePos = sampleUsePos;
         }
@@ -4346,7 +4354,7 @@ static int execInsert(threadInfo *winfo, char *buffer, int k)
   return affectedRows;
 }
 
-static int generateDataBuffer(int32_t threadID, threadInfo *pThreadInfo, char *buffer,
+static int generateDataBuffer(int32_t tableID, threadInfo *pThreadInfo, char *buffer,
         int64_t insertRows,
         int64_t startFrom, int64_t startTime, int *pSampleUsePos)
 {
@@ -4362,6 +4370,8 @@ static int generateDataBuffer(int32_t threadID, threadInfo *pThreadInfo, char *b
     }
   }
 
+  assert(buffer != NULL);
+
   memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len);
 
   char *pstr = buffer;
@@ -4374,7 +4384,7 @@ static int generateDataBuffer(int32_t threadID, threadInfo *pThreadInfo, char *b
       } else {
             tagsValBuf = getTagValueFromTagSample(
                     superTblInfo,
-                    threadID % superTblInfo->tagSampleCount);
+                    tableID % superTblInfo->tagSampleCount);
       }
       if (NULL == tagsValBuf) {
          fprintf(stderr, "tag buf failed to allocate  memory\n");
@@ -4386,7 +4396,7 @@ static int generateDataBuffer(int32_t threadID, threadInfo *pThreadInfo, char *b
                   "insert into %s.%s%d using %s.%s tags %s values",
                   pThreadInfo->db_name,
                   superTblInfo->childTblPrefix,
-                  threadID,
+                  tableID,
                   pThreadInfo->db_name,
                   superTblInfo->sTblName,
                   tagsValBuf);
@@ -4396,14 +4406,14 @@ static int generateDataBuffer(int32_t threadID, threadInfo *pThreadInfo, char *b
                   superTblInfo->maxSqlLen,
                   "insert into %s.%s values",
                   pThreadInfo->db_name,
-                  superTblInfo->childTblName + threadID * TSDB_TABLE_NAME_LEN);
+                  superTblInfo->childTblName + tableID * TSDB_TABLE_NAME_LEN);
     } else {
       pstr += snprintf(pstr,
                   (superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len),
                   "insert into %s.%s%d values",
                   pThreadInfo->db_name,
                   superTblInfo?superTblInfo->childTblPrefix:g_args.tb_prefix,
-                  threadID);
+                  tableID);
     }
   } else {
       pstr += snprintf(pstr,
@@ -4411,7 +4421,7 @@ static int generateDataBuffer(int32_t threadID, threadInfo *pThreadInfo, char *b
                   "insert into %s.%s%d values",
                   pThreadInfo->db_name,
                   superTblInfo?superTblInfo->childTblPrefix:g_args.tb_prefix,
-                  threadID);
+                  tableID);
   }
 
   int k;
@@ -4486,7 +4496,6 @@ static int generateDataBuffer(int32_t threadID, threadInfo *pThreadInfo, char *b
     k++;
     startFrom ++;
 
-    debugPrint("%s() LN%d k=%d startFrom=%ld insertRows=%ld\n", __func__, __LINE__, k, startFrom, insertRows);
     if (startFrom >= insertRows)
       break;
   }
@@ -4523,7 +4532,7 @@ static void* syncWrite(void *sarg) {
 
   char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1);
   if (NULL == buffer) {
-    fprintf(stderr, "Failed to calloc %d Bytes, reason:%s\n", 
+    fprintf(stderr, "Failed to alloc %d Bytes, reason:%s\n",
               superTblInfo->maxSqlLen,
               strerror(errno));
     tmfree(superTblInfo->sampleDataBuf);
@@ -4534,7 +4543,8 @@ static void* syncWrite(void *sarg) {
   int64_t startTs = taosGetTimestampUs();
   int64_t endTs;
 
-  int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
+  int insert_interval = superTblInfo?superTblInfo->insertInterval:
+      g_args.insert_interval;
   uint64_t st = 0;
   uint64_t et = 0xffffffff;
 
@@ -4543,12 +4553,12 @@ static void* syncWrite(void *sarg) {
 
   int sampleUsePos;
 
-  if (superTblInfo && superTblInfo->childTblLimit ) {
+  if (superTblInfo && superTblInfo->childTblOffset) {
       // TODO
   }
 
-  for (uint32_t tID = winfo->start_table_id; tID <= winfo->end_table_id;
-        tID++) {
+  for (uint32_t tableID = winfo->start_table_id; tableID <= winfo->end_table_id;
+        tableID++) {
     int64_t start_time = winfo->start_time;
 
     int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
@@ -4561,7 +4571,7 @@ static void* syncWrite(void *sarg) {
 
       sampleUsePos = samplePos;
 
-      int generated = generateDataBuffer(tID, winfo, buffer, insertRows,
+      int generated = generateDataBuffer(tableID, winfo, buffer, insertRows,
             i, start_time, &sampleUsePos);
       if (generated > 0)
         i += generated;
@@ -4605,12 +4615,12 @@ static void* syncWrite(void *sarg) {
       }
     }   // num_of_DPT
 
-    if ((tID == winfo->end_table_id) && superTblInfo &&
+    if ((tableID == winfo->end_table_id) && superTblInfo &&
         (0 == strncasecmp(
                     superTblInfo->dataSource, "sample", strlen("sample")))) {
           samplePos = sampleUsePos;
     }
-  } // tID
+  } // tableID
 
 free_and_statistics_2:
   tmfree(buffer);
@@ -4705,12 +4715,19 @@ static void startMultiThreadInsertData(int threads, char* db_name,
         char* precision,SSuperTable* superTblInfo) {
 
     pthread_t *pids = malloc(threads * sizeof(pthread_t));
+    assert(pids != NULL);
+
     threadInfo *infos = malloc(threads * sizeof(threadInfo));
+    assert(infos != NULL);
+
     memset(pids, 0, threads * sizeof(pthread_t));
     memset(infos, 0, threads * sizeof(threadInfo));
 
     int ntables = 0;
     if (superTblInfo)
+        if (superTblInfo->childTblOffset)
+            ntables = superTblInfo->childTblLimit;
+        else
             ntables = superTblInfo->childTblCount;
     else
         ntables = g_args.num_of_tables;
@@ -4767,7 +4784,13 @@ static void startMultiThreadInsertData(int threads, char* db_name,
 
   double start = getCurrentTime();
  
-  int last = 0;
+  int last;
+
+  if ((superTblInfo) && (superTblInfo->childTblOffset))
+      last = superTblInfo->childTblOffset;
+  else
+      last = 0;
+
   for (int i = 0; i < threads; i++) {
     threadInfo *t_info = infos + i;
     t_info->threadID = i;