taosdemo.c 245.1 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

21
#include <stdint.h>
S
slguan 已提交
22
#define _GNU_SOURCE
23 24
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
41
#else
S
TD-1057  
Shengliang Guan 已提交
42 43
  #include <regex.h>
  #include <stdio.h>
44
#endif
H
hzcheng 已提交
45

46
#include <assert.h>
47
#include <stdlib.h>
48 49
#include "cJSON.h"

50
#include "os.h"
H
hzcheng 已提交
51
#include "taos.h"
52
#include "taoserror.h"
H
Hui Li 已提交
53
#include "tutil.h"
S
slguan 已提交
54

55 56 57
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
58 59
extern char configDir[];

60 61 62 63
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

64 65 66 67 68 69
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};
70

71 72
#define MAX_RECORDS_PER_REQ     32766

73 74
#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
75
#define COND_BUF_LEN        BUFFER_SIZE - 30
76 77
#define MAX_USERNAME_SIZE  64
#define MAX_PASSWORD_SIZE  64
78
#define MAX_DB_NAME_SIZE   64
79
#define MAX_HOSTNAME_SIZE  64
80
#define MAX_TB_NAME_SIZE   64
81
#define MAX_DATA_SIZE      (16*1024)+20     // max record len: 16*1024, timestamp string and ,('') need extra space
82 83 84 85 86 87 88 89 90 91
#define MAX_NUM_DATATYPE   10
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
#define MAX_FILE_NAME_LEN  256

#define   MAX_SAMPLES_ONCE_FROM_FILE   10000
#define   MAX_NUM_DATATYPE 10

#define   MAX_DB_COUNT           8
92
#define   MAX_SUPER_TABLE_COUNT  200
93 94 95
#define   MAX_COLUMN_COUNT       1024
#define   MAX_TAG_COUNT          128

96
#define   MAX_QUERY_SQL_COUNT    100
97
#define   MAX_QUERY_SQL_LENGTH   1024
98 99

#define   MAX_DATABASE_COUNT     256
100
#define INPUT_BUF_LEN   256
101

102
#define DEFAULT_TIMESTAMP_STEP  1
103

104

105 106 107 108 109
typedef enum CREATE_SUB_TALBE_MOD_EN {
  PRE_CREATE_SUBTBL,
  AUTO_CREATE_SUBTBL,
  NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
110

111 112
typedef enum TALBE_EXISTS_EN {
  TBL_NO_EXISTS,
113
  TBL_ALREADY_EXISTS,
114 115 116
  TBL_EXISTS_BUTT
} TALBE_EXISTS_EN;

117
enum enumSYNC_MODE {
118 119
  SYNC_MODE,
  ASYNC_MODE,
120 121
  MODE_BUT
};
122

123 124 125 126 127 128
typedef enum enumQUERY_CLASS {
    SPECIFIED_CLASS,
    STABLE_CLASS,
    CLASS_BUT
} QUERY_CLASS;

129 130 131 132 133 134
typedef enum enum_INSERT_MODE {
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
} INSERT_MODE;

135
typedef enum enumQUERY_TYPE {
136
  NO_INSERT_TYPE,
137
  INSERT_TYPE,
138
  QUERY_TYPE_BUT
139
} QUERY_TYPE;
140 141 142 143 144 145 146

enum _show_db_index {
  TSDB_SHOW_DB_NAME_INDEX,
  TSDB_SHOW_DB_CREATED_TIME_INDEX,
  TSDB_SHOW_DB_NTABLES_INDEX,
  TSDB_SHOW_DB_VGROUPS_INDEX,
  TSDB_SHOW_DB_REPLICA_INDEX,
147
  TSDB_SHOW_DB_QUORUM_INDEX,
148 149 150 151 152 153 154 155 156 157
  TSDB_SHOW_DB_DAYS_INDEX,
  TSDB_SHOW_DB_KEEP_INDEX,
  TSDB_SHOW_DB_CACHE_INDEX,
  TSDB_SHOW_DB_BLOCKS_INDEX,
  TSDB_SHOW_DB_MINROWS_INDEX,
  TSDB_SHOW_DB_MAXROWS_INDEX,
  TSDB_SHOW_DB_WALLEVEL_INDEX,
  TSDB_SHOW_DB_FSYNC_INDEX,
  TSDB_SHOW_DB_COMP_INDEX,
  TSDB_SHOW_DB_CACHELAST_INDEX,
158
  TSDB_SHOW_DB_PRECISION_INDEX,
159 160 161 162 163 164 165 166 167 168
  TSDB_SHOW_DB_UPDATE_INDEX,
  TSDB_SHOW_DB_STATUS_INDEX,
  TSDB_MAX_SHOW_DB
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
  TSDB_SHOW_STABLES_NAME_INDEX,
  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
  TSDB_SHOW_STABLES_COLUMNS_INDEX,
169 170
  TSDB_SHOW_STABLES_METRIC_INDEX,
  TSDB_SHOW_STABLES_UID_INDEX,
171
  TSDB_SHOW_STABLES_TID_INDEX,
172
  TSDB_SHOW_STABLES_VGID_INDEX,
173
  TSDB_MAX_SHOW_STABLES
174 175
};

176 177 178 179 180 181 182 183 184 185 186 187 188 189
enum _describe_table_index {
  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
  TSDB_MAX_DESCRIBE_METRIC
};

typedef struct {
  char field[TSDB_COL_NAME_LEN + 1];
  char type[16];
  int length;
  char note[128];
} SColDes;
H
hzcheng 已提交
190 191

/* Used by main to communicate with parse_opt. */
192 193
static char *g_dupstr = NULL;

194 195
typedef struct SArguments_S {
  char *   metaFile;
196
  uint32_t test_mode;
197 198 199 200 201 202 203 204 205
  char *   host;
  uint16_t port;
  char *   user;
  char *   password;
  char *   database;
  int      replica;
  char *   tb_prefix;
  char *   sqlFile;
  bool     use_metric;
206
  bool     drop_database;
207
  bool     insert_only;
208
  bool     answer_yes;
209
  bool     debug_print;
210
  bool     verbose_print;
211
  bool     performance_print;
212
  char *   output_file;
213
  bool     async_mode;
214
  char *   datatype[MAX_NUM_DATATYPE + 1];
215 216 217 218
  uint32_t len_of_binary;
  uint32_t num_of_CPR;
  uint32_t num_of_threads;
  uint64_t insert_interval;
219
  int64_t  query_times;
220 221 222
  uint64_t interlace_rows;
  uint64_t num_of_RPR;                  // num_of_records_per_req
  uint64_t max_sql_len;
223 224
  int64_t  num_of_tables;
  int64_t  num_of_DPT;
225
  int      abort;
226 227
  int      disorderRatio;               // 0: no disorder, >0: x%
  int      disorderRange;               // ms or us by database precision
228
  uint32_t method_of_delete;
229
  char **  arg_list;
230 231
  uint64_t totalInsertRows;
  uint64_t totalAffectedRows;
232 233 234
} SArguments;

typedef struct SColumn_S {
235 236 237
  char      field[TSDB_COL_NAME_LEN + 1];
  char      dataType[MAX_TB_NAME_SIZE];
  uint32_t  dataLen;
238
  char      note[128];
239 240 241
} StrColumn;

typedef struct SSuperTable_S {
242
  char         sTblName[MAX_TB_NAME_SIZE+1];
243
  int64_t      childTblCount;
244
  bool         childTblExists;          // 0: no, 1: yes
245 246
  uint64_t     batchCreateTableNum;     // 0: no batch,  > 0: batch table number in one sql
  uint8_t      autoCreateTable;         // 0: create sub table, 1: auto create sub table
247
  char         childTblPrefix[MAX_TB_NAME_SIZE];
248
  char         dataSource[MAX_TB_NAME_SIZE+1];  // rand_gen or sample
249
  char         insertMode[MAX_TB_NAME_SIZE];    // taosc, rest
250
  int64_t      childTblLimit;
251
  uint64_t     childTblOffset;
252

253
//  int          multiThreadWriteOneTbl;  // 0: no, 1: yes
254
  uint64_t     interlaceRows;           //
255 256
  int          disorderRatio;           // 0: no disorder, >0: x%
  int          disorderRange;           // ms or us by database precision
257
  uint64_t     maxSqlLen;               //
258

259
  uint64_t     insertInterval;          // insert interval, will override global insert interval
260
  int64_t      insertRows;
261
  int64_t      timeStampStep;
262
  char         startTimestamp[MAX_TB_NAME_SIZE];
263
  char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
264 265
  char         sampleFile[MAX_FILE_NAME_LEN+1];
  char         tagsFile[MAX_FILE_NAME_LEN+1];
266

267
  uint32_t     columnCount;
268
  StrColumn    columns[MAX_COLUMN_COUNT];
269
  uint32_t     tagCount;
270 271 272
  StrColumn    tags[MAX_TAG_COUNT];

  char*        childTblName;
273
  char*        colsOfCreateChildTable;
274 275
  uint64_t     lenOfOneRow;
  uint64_t     lenOfTagOfOneRow;
276 277 278 279 280

  char*        sampleDataBuf;
  //int          sampleRowCount;
  //int          sampleUsePos;

281
  uint32_t     tagSource;    // 0: rand, 1: tag sample
282
  char*        tagDataBuf;
283 284
  uint32_t     tagSampleCount;
  uint32_t     tagUsePos;
285

286
  // statistics
287 288
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
289 290 291 292 293
} SSuperTable;

typedef struct {
  char     name[TSDB_DB_NAME_LEN + 1];
  char     create_time[32];
294
  int64_t  ntables;
295
  int32_t  vgroups;
296 297
  int16_t  replica;
  int16_t  quorum;
298
  int16_t  days;
299 300 301 302 303 304 305 306 307 308 309 310 311 312
  char     keeplist[32];
  int32_t  cache; //MB
  int32_t  blocks;
  int32_t  minrows;
  int32_t  maxrows;
  int8_t   wallevel;
  int32_t  fsync;
  int8_t   comp;
  int8_t   cachelast;
  char     precision[8];   // time resolution
  int8_t   update;
  char     status[16];
} SDbInfo;

313
typedef struct SDbCfg_S {
314
//  int       maxtablesPerVnode;
315 316
  uint32_t  minRows;        // 0 means default
  uint32_t  maxRows;        // 0 means default
317 318
  int       comp;
  int       walLevel;
319
  int       cacheLast;
320
  int       fsync;
321 322 323 324 325 326 327
  int       replica;
  int       update;
  int       keep;
  int       days;
  int       cache;
  int       blocks;
  int       quorum;
328
  char      precision[MAX_TB_NAME_SIZE];
329 330 331 332
} SDbCfg;

typedef struct SDataBase_S {
  char         dbName[MAX_DB_NAME_SIZE];
333
  bool         drop;  // 0: use exists, 1: if exists, drop then new create
334
  SDbCfg       dbCfg;
335
  uint64_t     superTblCount;
336 337 338 339
  SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
} SDataBase;

typedef struct SDbs_S {
340
  char         cfgDir[MAX_FILE_NAME_LEN+1];
341
  char         host[MAX_HOSTNAME_SIZE];
342 343
  struct sockaddr_in serv_addr;

344
  uint16_t     port;
345 346
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
347
  char         resultFile[MAX_FILE_NAME_LEN+1];
348 349 350
  bool         use_metric;
  bool         insert_only;
  bool         do_aggreFunc;
351
  bool         asyncMode;
352

353 354 355
  uint32_t     threadCount;
  uint32_t     threadCountByCreateTbl;
  uint32_t     dbCount;
356 357 358
  SDataBase    db[MAX_DB_COUNT];

  // statistics
359 360
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
361

362 363
} SDbs;

364
typedef struct SpecifiedQueryInfo_S {
365
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
366
  uint32_t     concurrent;
367
  uint64_t     sqlCount;
368
  uint32_t     asyncMode; // 0: sync, 1: async
369 370
  uint64_t     subscribeInterval; // ms
  uint64_t     queryTimes;
371
  bool         subscribeRestart;
372
  int          subscribeKeepProgress;
373
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
374
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
375
  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
376
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
377 378 379
  char         topic[MAX_QUERY_SQL_COUNT][32];
  int          consumed[MAX_QUERY_SQL_COUNT];
  TAOS_RES*    res[MAX_QUERY_SQL_COUNT];
380
  uint64_t     totalQueried;
381
} SpecifiedQueryInfo;
382

383
typedef struct SuperQueryInfo_S {
384
  char         sTblName[MAX_TB_NAME_SIZE+1];
385 386
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint32_t     threadCnt;
387
  uint32_t     asyncMode; // 0: sync, 1: async
388
  uint64_t     subscribeInterval; // ms
389
  bool         subscribeRestart;
390
  int          subscribeKeepProgress;
391
  uint64_t     queryTimes;
392
  int64_t      childTblCount;
393
  char         childTblPrefix[MAX_TB_NAME_SIZE];
394
  uint64_t     sqlCount;
395
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
396
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
397
  int          resubAfterConsume;
398
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
399

400
  char*        childTblName;
401
  uint64_t     totalQueried;
402
} SuperQueryInfo;
403 404

typedef struct SQueryMetaInfo_S {
405
  char         cfgDir[MAX_FILE_NAME_LEN+1];
406
  char         host[MAX_HOSTNAME_SIZE];
407
  uint16_t     port;
408
  struct       sockaddr_in serv_addr;
409 410
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
411
  char         dbName[MAX_DB_NAME_SIZE+1];
412
  char         queryMode[MAX_TB_NAME_SIZE];  // taosc, rest
413

414
  SpecifiedQueryInfo  specifiedQueryInfo;
415
  SuperQueryInfo      superQueryInfo;
416
  uint64_t     totalQueried;
417 418 419
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
420 421 422 423
  TAOS *    taos;
  int       threadID;
  char      db_name[MAX_DB_NAME_SIZE+1];
  uint32_t  time_precision;
424 425
  char      filePath[4096];
  FILE      *fp;
426 427 428
  char      tb_prefix[MAX_TB_NAME_SIZE];
  uint64_t  start_table_from;
  uint64_t  end_table_to;
429
  int64_t   ntables;
430 431 432 433
  uint64_t  data_of_rate;
  int64_t   start_time;
  char*     cols;
  bool      use_metric;
434 435 436
  SSuperTable* superTblInfo;

  // for async insert
437 438
  tsem_t    lock_sem;
  int64_t   counter;
439 440
  uint64_t  st;
  uint64_t  et;
441
  uint64_t  lastTs;
442

443
  // sample data
444
  int64_t   samplePos;
445
  // statistics
446 447
  uint64_t  totalInsertRows;
  uint64_t  totalAffectedRows;
448 449

  // insert delay statistics
450 451 452 453 454
  uint64_t  cntDelay;
  uint64_t  totalDelay;
  uint64_t  avgDelay;
  uint64_t  maxDelay;
  uint64_t  minDelay;
455

456
  // seq of query or subscribe
457
  uint64_t  querySeq;   // sequence number of sql command
458
  TAOS_SUB*  tsub;
459

460 461
} threadInfo;

462
#ifdef WINDOWS
463 464
#define _CRT_RAND_S

465
#include <windows.h>
466 467 468 469 470
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
471 472
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
473 474
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
475 476 477 478

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

479
static void setupForAnsiEscape(void) {
480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
498
  }
499 500
}

501
static void resetAfterAnsiEscape(void) {
502
  // Reset colors
503
  printf("\x1b[0m");
504 505 506 507 508 509

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
510

511
static int taosRandom()
512 513 514 515 516 517
{
    int number;
    rand_s(&number);

    return number;
}
518
#else   // Not windows
519
static void setupForAnsiEscape(void) {}
520

521
static void resetAfterAnsiEscape(void) {
522 523 524
  // Reset colors
  printf("\x1b[0m");
}
525

526 527
#include <time.h>

528
static int taosRandom()
529
{
530
  return rand();
531 532
}

533
#endif // ifdef Windows
534

535
static void prompt();
536
static int createDatabasesAndStables();
537
static void createChildTables();
538
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet);
539
static int postProceSql(char *host, struct sockaddr_in *pServAddr,
540
        uint16_t port, char* sqlstr, threadInfo *pThreadInfo);
541 542 543 544 545 546 547

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
548
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)",
549 550
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

551
SArguments g_args = {
552 553
                     NULL,            // metaFile
                     0,               // test_mode
554 555 556
                     "127.0.0.1",     // host
                     6030,            // port
                     "root",          // user
557
                     #ifdef _TD_POWER_
558 559 560 561 562 563 564 565
                     "powerdb",      // password
                     #else
                     "taosdata",      // password
                     #endif
                     "test",          // database
                     1,               // replica
                     "t",             // tb_prefix
                     NULL,            // sqlFile
566
                     true,            // use_metric
567
                     true,            // drop_database
568
                     true,            // insert_only
569
                     false,           // debug_print
570
                     false,           // verbose_print
571
                     false,           // performance statistic print
572 573 574 575
                     false,           // answer_yes;
                     "./output.txt",  // output_file
                     0,               // mode : sync or async
                     {
576 577 578 579
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
580 581
                     },
                     16,              // len_of_binary
582
                     4,               // num_of_CPR
583 584
                     10,              // num_of_connections/thread
                     0,               // insert_interval
585
                     1,               // query_times
586
                     0,               // interlace_rows;
587
                     30000,           // num_of_RPR
588
                     (1024*1024),         // max_sql_len
589 590 591 592 593 594 595 596 597 598 599 600
                     10000,           // num_of_tables
                     10000,           // num_of_DPT
                     0,               // abort
                     0,               // disorderRatio
                     1000,            // disorderRange
                     1,               // method_of_delete
                     NULL             // arg_list
};



static SDbs            g_Dbs;
601
static int64_t         g_totalChildTables = 0;
602 603 604 605
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

#define debugPrint(fmt, ...) \
606
    do { if (g_args.debug_print || g_args.verbose_print) \
607
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
608

609
#define verbosePrint(fmt, ...) \
610 611
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
612

613 614 615 616
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)

617 618 619 620
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)


621
///////////////////////////////////////////////////
H
hzcheng 已提交
622

623 624
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

625 626 627 628 629 630 631 632
#ifndef TAOSDEMO_COMMIT_SHA1
#define TAOSDEMO_COMMIT_SHA1 "unknown"
#endif

#ifndef TD_VERNUMBER
#define TD_VERNUMBER    "unknown"
#endif

633
#ifndef TAOSDEMO_STATUS
634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650
#define TAOSDEMO_STATUS "unknown"
#endif

static void printVersion() {
    char tdengine_ver[] = TD_VERNUMBER;
    char taosdemo_ver[] = TAOSDEMO_COMMIT_SHA1;
    char taosdemo_status[] = TAOSDEMO_STATUS;

    if (strlen(taosdemo_status) == 0) {
        printf("taosdemo verison %s-%s\n",
                tdengine_ver, taosdemo_ver);
    } else {
        printf("taosdemo verison %s-%s, status:%s\n",
                tdengine_ver, taosdemo_ver, taosdemo_status);
    }
}

651
static void printHelp() {
652
  char indent[10] = "        ";
653
  printf("%s%s%s%s\n", indent, "-f", indent,
654
          "The meta file to the execution procedure. Default is './meta.json'.");
655
  printf("%s%s%s%s\n", indent, "-u", indent,
656 657
          "The TDengine user name to use when connecting to the server. Default is 'root'.");
#ifdef _TD_POWER_
658
  printf("%s%s%s%s\n", indent, "-P", indent,
659
          "The password to use when connecting to the server. Default is 'powerdb'.");
660
  printf("%s%s%s%s\n", indent, "-c", indent,
661 662
          "Configuration directory. Default is '/etc/power/'.");
#else
663
  printf("%s%s%s%s\n", indent, "-P", indent,
664
          "The password to use when connecting to the server. Default is 'taosdata'.");
665
  printf("%s%s%s%s\n", indent, "-c", indent,
666
          "Configuration directory. Default is '/etc/taos/'.");
667
#endif
668
  printf("%s%s%s%s\n", indent, "-h", indent,
669
          "The host to connect to TDengine. Default is localhost.");
670
  printf("%s%s%s%s\n", indent, "-p", indent,
671
          "The TCP/IP port number to use for the connection. Default is 0.");
672
  printf("%s%s%s%s\n", indent, "-d", indent,
673
          "Destination database. Default is 'test'.");
674
  printf("%s%s%s%s\n", indent, "-a", indent,
675
          "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
676
  printf("%s%s%s%s\n", indent, "-m", indent,
677 678
          "Table prefix name. Default is 't'.");
  printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
679
  printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
680
  printf("%s%s%s%s\n", indent, "-o", indent,
681
          "Direct output to the named file. Default is './output.txt'.");
682
  printf("%s%s%s%s\n", indent, "-q", indent,
683
          "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC.");
684
  printf("%s%s%s%s\n", indent, "-b", indent,
685
          "The data_type of columns, default: INT,INT,INT,INT.");
686
  printf("%s%s%s%s\n", indent, "-w", indent,
687
          "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
688 689 690
  printf("%s%s%s%s%d\n", indent, "-l", indent,
          "The number of columns per record. Default is 4. Max values is ",
       MAX_NUM_DATATYPE);
691
  printf("%s%s%s%s\n", indent, "-T", indent,
692
          "The number of threads. Default is 10.");
693
  printf("%s%s%s%s\n", indent, "-i", indent,
694
          "The sleep time (ms) between insertion. Default is 0.");
695
  printf("%s%s%s%s\n", indent, "-r", indent,
696
          "The number of records per request. Default is 30000.");
697
  printf("%s%s%s%s\n", indent, "-t", indent,
698
          "The number of tables. Default is 10000.");
699
  printf("%s%s%s%s\n", indent, "-n", indent,
700 701 702
          "The number of records per table. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
  printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
703
  printf("%s%s%s%s\n", indent, "-O", indent,
704
          "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order.");
705
  printf("%s%s%s%s\n", indent, "-R", indent,
706
          "Out of order data's range, ms, default is 1000.");
707
  printf("%s%s%s%s\n", indent, "-g", indent,
708
          "Print debug info.");
709
  printf("%s%s%s\n", indent, "-V, --version\t",
710
          "Print version info.");
711 712
  printf("%s%s%s%s\n", indent, "--help\t", indent,
          "Print command line arguments list info.");
713
/*    printf("%s%s%s%s\n", indent, "-D", indent,
714 715 716 717
          "if elete database if exists. 0: no, 1: yes, default is 1");
          */
}

718 719 720 721 722 723 724 725 726 727 728 729 730 731 732
static bool isStringNumber(char *input)
{
  int len = strlen(input);
  if (0 == len) {
    return false;
  }

  for (int i = 0; i < len; i++) {
    if (!isdigit(input[i]))
      return false;
  }

  return true;
}

733
static void parse_args(int argc, char *argv[], SArguments *arguments) {
734 735 736 737 738

  for (int i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-f") == 0) {
      arguments->metaFile = argv[++i];
    } else if (strcmp(argv[i], "-c") == 0) {
739 740 741 742 743
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-c need a valid path following!\n");
        exit(EXIT_FAILURE);
      }
744
      tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN);
745
    } else if (strcmp(argv[i], "-h") == 0) {
746 747 748 749 750
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-h need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
751 752
      arguments->host = argv[++i];
    } else if (strcmp(argv[i], "-p") == 0) {
753 754 755 756 757 758
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-p need a number following!\n");
        exit(EXIT_FAILURE);
      }
759 760
      arguments->port = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-u") == 0) {
761 762 763 764 765
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-u need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
766 767
      arguments->user = argv[++i];
    } else if (strcmp(argv[i], "-P") == 0) {
768 769 770 771 772
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-P need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
773 774
      arguments->password = argv[++i];
    } else if (strcmp(argv[i], "-o") == 0) {
775 776 777 778 779
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-o need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
780 781
      arguments->output_file = argv[++i];
    } else if (strcmp(argv[i], "-s") == 0) {
782 783 784 785 786
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-s need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
787 788
      arguments->sqlFile = argv[++i];
    } else if (strcmp(argv[i], "-q") == 0) {
789 790
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
791
        printHelp();
792
        errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n");
793 794
        exit(EXIT_FAILURE);
      }
795
      arguments->async_mode = atoi(argv[++i]);
796
    } else if (strcmp(argv[i], "-T") == 0) {
797 798
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
799 800 801 802
        printHelp();
        errorPrint("%s", "\n\t-T need a number following!\n");
        exit(EXIT_FAILURE);
      }
803 804
      arguments->num_of_threads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
805 806
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
807 808 809 810
        printHelp();
        errorPrint("%s", "\n\t-i need a number following!\n");
        exit(EXIT_FAILURE);
      }
811
      arguments->insert_interval = atoi(argv[++i]);
812
    } else if (strcmp(argv[i], "-qt") == 0) {
813 814
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
815
        printHelp();
816
        errorPrint("%s", "\n\t-qt need a number following!\n");
817 818
        exit(EXIT_FAILURE);
      }
819
      arguments->query_times = atoi(argv[++i]);
820
    } else if (strcmp(argv[i], "-B") == 0) {
821 822
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
823 824 825 826
        printHelp();
        errorPrint("%s", "\n\t-B need a number following!\n");
        exit(EXIT_FAILURE);
      }
827
      arguments->interlace_rows = atoi(argv[++i]);
828
    } else if (strcmp(argv[i], "-r") == 0) {
829 830
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
831 832 833 834
        printHelp();
        errorPrint("%s", "\n\t-r need a number following!\n");
        exit(EXIT_FAILURE);
      }
835 836
      arguments->num_of_RPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-t") == 0) {
837 838 839 840 841 842
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-t need a number following!\n");
        exit(EXIT_FAILURE);
      }
843 844
      arguments->num_of_tables = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
845 846 847 848 849 850
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-n need a number following!\n");
        exit(EXIT_FAILURE);
      }
851 852
      arguments->num_of_DPT = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-d") == 0) {
853 854 855 856 857
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-d need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
858 859
      arguments->database = argv[++i];
    } else if (strcmp(argv[i], "-l") == 0) {
860 861 862 863 864 865
      if (argc == i+1) {
        if (!isStringNumber(argv[i+1])) {
            printHelp();
            errorPrint("%s", "\n\t-l need a number following!\n");
            exit(EXIT_FAILURE);
        }
866
      }
867
      arguments->num_of_CPR = atoi(argv[++i]);
868 869 870 871 872 873 874 875 876 877 878

      if (arguments->num_of_CPR > MAX_NUM_DATATYPE) {
          printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE);
          prompt();
          arguments->num_of_CPR = MAX_NUM_DATATYPE;
      }

      for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) {
          arguments->datatype[col] = NULL;
      }

879
    } else if (strcmp(argv[i], "-b") == 0) {
880 881 882 883 884
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-b need valid string following!\n");
        exit(EXIT_FAILURE);
      }
885 886
      ++i;
      if (strstr(argv[i], ",") == NULL) {
887
        // only one col
888 889 890 891 892 893 894
        if (strcasecmp(argv[i], "INT")
                && strcasecmp(argv[i], "FLOAT")
                && strcasecmp(argv[i], "TINYINT")
                && strcasecmp(argv[i], "BOOL")
                && strcasecmp(argv[i], "SMALLINT")
                && strcasecmp(argv[i], "BIGINT")
                && strcasecmp(argv[i], "DOUBLE")
895
                && strcasecmp(argv[i], "BINARY")
896 897
                && strcasecmp(argv[i], "NCHAR")) {
          printHelp();
898
          errorPrint("%s", "-b: Invalid data_type!\n");
899
          exit(EXIT_FAILURE);
900
        }
901
        arguments->datatype[0] = argv[i];
902
      } else {
903
        // more than one col
904
        int index = 0;
905 906
        g_dupstr = strdup(argv[i]);
        char *running = g_dupstr;
907
        char *token = strsep(&running, ",");
908
        while(token != NULL) {
909 910 911 912 913 914 915 916 917 918
          if (strcasecmp(token, "INT")
                  && strcasecmp(token, "FLOAT")
                  && strcasecmp(token, "TINYINT")
                  && strcasecmp(token, "BOOL")
                  && strcasecmp(token, "SMALLINT")
                  && strcasecmp(token, "BIGINT")
                  && strcasecmp(token, "DOUBLE")
                  && strcasecmp(token, "BINARY")
                  && strcasecmp(token, "NCHAR")) {
            printHelp();
919
            free(g_dupstr);
920
            errorPrint("%s", "-b: Invalid data_type!\n");
921
            exit(EXIT_FAILURE);
H
hzcheng 已提交
922
          }
923
          arguments->datatype[index++] = token;
924 925
          token = strsep(&running, ",");
          if (index >= MAX_NUM_DATATYPE) break;
S
TD-1057  
Shengliang Guan 已提交
926
        }
927
        arguments->datatype[index] = NULL;
928
      }
929
    } else if (strcmp(argv[i], "-w") == 0) {
930 931 932 933 934 935
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-w need a number following!\n");
        exit(EXIT_FAILURE);
      }
936 937
      arguments->len_of_binary = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-m") == 0) {
938 939 940 941 942 943
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-m need a number following!\n");
        exit(EXIT_FAILURE);
      }
944
      arguments->tb_prefix = argv[++i];
945 946
    } else if (strcmp(argv[i], "-N") == 0) {
      arguments->use_metric = false;
947
    } else if (strcmp(argv[i], "-x") == 0) {
948
      arguments->insert_only = false;
949
    } else if (strcmp(argv[i], "-y") == 0) {
950
      arguments->answer_yes = true;
951 952
    } else if (strcmp(argv[i], "-g") == 0) {
      arguments->debug_print = true;
953 954
    } else if (strcmp(argv[i], "-gg") == 0) {
      arguments->verbose_print = true;
955 956
    } else if (strcmp(argv[i], "-pp") == 0) {
      arguments->performance_print = true;
957
    } else if (strcmp(argv[i], "-O") == 0) {
958 959 960 961 962 963
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-O need a number following!\n");
        exit(EXIT_FAILURE);
      }
964

965
      arguments->disorderRatio = atoi(argv[++i]);
966

967
      if (arguments->disorderRatio > 50) {
968
        arguments->disorderRatio = 50;
969
      }
970

971
      if (arguments->disorderRatio < 0) {
972
        arguments->disorderRatio = 0;
973
      }
974

975
    } else if (strcmp(argv[i], "-R") == 0) {
976 977 978 979 980 981
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-R need a number following!\n");
        exit(EXIT_FAILURE);
      }
982

983
      arguments->disorderRange = atoi(argv[++i]);
984 985 986
      if (arguments->disorderRange < 0)
        arguments->disorderRange = 1000;

987
    } else if (strcmp(argv[i], "-a") == 0) {
988 989 990 991 992 993
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-a need a number following!\n");
        exit(EXIT_FAILURE);
      }
994 995 996
      arguments->replica = atoi(argv[++i]);
      if (arguments->replica > 3 || arguments->replica < 1) {
          arguments->replica = 1;
997
      }
998 999
    } else if (strcmp(argv[i], "-D") == 0) {
      arguments->method_of_delete = atoi(argv[++i]);
1000 1001 1002
      if (arguments->method_of_delete > 3) {
        errorPrint("%s", "\n\t-D need a valud (0~3) number following!\n");
        exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1003
      }
1004 1005 1006 1007
    } else if ((strcmp(argv[i], "--version") == 0) ||
        (strcmp(argv[i], "-V") == 0)){
      printVersion();
      exit(0);
1008 1009 1010 1011 1012
    } else if (strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else {
      printHelp();
1013
      errorPrint("%s", "ERROR: wrong options\n");
1014
      exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1015 1016
    }
  }
1017

1018 1019
  if (((arguments->debug_print) && (arguments->metaFile == NULL))
          || arguments->verbose_print) {
1020
    printf("###################################################################\n");
1021
    printf("# meta file:                         %s\n", arguments->metaFile);
1022
    printf("# Server IP:                         %s:%hu\n",
1023 1024 1025 1026 1027
            arguments->host == NULL ? "localhost" : arguments->host,
            arguments->port );
    printf("# User:                              %s\n", arguments->user);
    printf("# Password:                          %s\n", arguments->password);
    printf("# Use metric:                        %s\n", arguments->use_metric ? "true" : "false");
1028 1029 1030 1031 1032 1033 1034 1035 1036
    if (*(arguments->datatype)) {
        printf("# Specified data type:               ");
        for (int i = 0; i < MAX_NUM_DATATYPE; i++)
            if (arguments->datatype[i])
               printf("%s,", arguments->datatype[i]);
            else
                break;
        printf("\n");
    }
1037
    printf("# Insertion interval:                %"PRIu64"\n",
1038
            arguments->insert_interval);
1039
    printf("# Number of records per req:         %"PRIu64"\n",
1040
            arguments->num_of_RPR);
1041
    printf("# Max SQL length:                    %"PRIu64"\n",
1042
            arguments->max_sql_len);
1043
    printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
1044
    printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
1045
    printf("# Number of Tables:                  %"PRId64"\n",
1046
            arguments->num_of_tables);
1047
    printf("# Number of Data per Table:          %"PRId64"\n",
1048
            arguments->num_of_DPT);
1049 1050 1051 1052 1053
    printf("# Database name:                     %s\n", arguments->database);
    printf("# Table prefix:                      %s\n", arguments->tb_prefix);
    if (arguments->disorderRatio) {
      printf("# Data order:                        %d\n", arguments->disorderRatio);
      printf("# Data out of order rate:            %d\n", arguments->disorderRange);
1054

1055 1056 1057 1058
    }
    printf("# Delete method:                     %d\n", arguments->method_of_delete);
    printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
    printf("# Print debug info:                  %d\n", arguments->debug_print);
1059
    printf("# Print verbose info:                %d\n", arguments->verbose_print);
1060
    printf("###################################################################\n");
1061 1062

    prompt();
1063
  }
1064
}
H
hzcheng 已提交
1065

1066 1067 1068 1069
static bool getInfoFromJsonFile(char* file);
//static int generateOneRowDataForStb(SSuperTable* stbInfo);
//static int getDataIntoMemForStb(SSuperTable* stbInfo);
static void init_rand_data();
1070
static void tmfclose(FILE *fp) {
1071 1072 1073 1074
  if (NULL != fp) {
    fclose(fp);
  }
}
1075

1076
static void tmfree(char *buf) {
1077 1078
  if (NULL != buf) {
    free(buf);
H
Hui Li 已提交
1079
  }
1080
}
H
Hui Li 已提交
1081

1082
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
1083 1084 1085
  int i;
  TAOS_RES *res = NULL;
  int32_t   code = -1;
1086

1087 1088 1089 1090
  for (i = 0; i < 5; i++) {
    if (NULL != res) {
      taos_free_result(res);
      res = NULL;
H
hzcheng 已提交
1091
    }
1092

1093 1094 1095 1096
    res = taos_query(taos, command);
    code = taos_errno(res);
    if (0 == code) {
      break;
1097
    }
H
hzcheng 已提交
1098 1099
  }

1100
  if (code != 0) {
1101 1102
    if (!quiet) {
      debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
1103
      errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res));
1104
    }
1105 1106 1107
    taos_free_result(res);
    //taos_close(taos);
    return -1;
1108
  }
H
hzcheng 已提交
1109

1110 1111 1112 1113
  if (INSERT_TYPE == type) {
    int affectedRows = taos_affected_rows(res);
    taos_free_result(res);
    return affectedRows;
H
hzcheng 已提交
1114
  }
1115

1116
  taos_free_result(res);
1117 1118
  return 0;
}
H
hzcheng 已提交
1119

1120
static void appendResultBufToFile(char *resultBuf, threadInfo *pThreadInfo)
1121
{
1122 1123
  pThreadInfo->fp = fopen(pThreadInfo->filePath, "at");
  if (pThreadInfo->fp == NULL) {
1124 1125
      errorPrint(
              "%s() LN%d, failed to open result file: %s, result will not save to file\n",
1126
              __func__, __LINE__, pThreadInfo->filePath);
1127 1128
      return;
  }
1129

1130 1131 1132
  fprintf(pThreadInfo->fp, "%s", resultBuf);
  tmfclose(pThreadInfo->fp);
  pThreadInfo->fp = NULL;
1133
}
1134

1135
static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) {
1136 1137 1138 1139 1140
  TAOS_ROW    row = NULL;
  int         num_rows = 0;
  int         num_fields = taos_field_count(res);
  TAOS_FIELD *fields     = taos_fetch_fields(res);

1141 1142
  char* databuf = (char*) calloc(1, 100*1024*1024);
  if (databuf == NULL) {
1143 1144
    errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n",
            __func__, __LINE__);
1145 1146
    return ;
  }
H
hzcheng 已提交
1147

1148 1149
  int   totalLen = 0;
  char  temp[16000];
H
hzcheng 已提交
1150

1151
  // fetch the records row by row
1152
  while((row = taos_fetch_row(res))) {
1153
    if ((strlen(pThreadInfo->filePath) > 0)
1154
            && (totalLen >= 100*1024*1024 - 32000)) {
1155
        appendResultBufToFile(databuf, pThreadInfo);
1156 1157
        totalLen = 0;
        memset(databuf, 0, 100*1024*1024);
H
hzcheng 已提交
1158
    }
1159 1160 1161 1162 1163 1164
    num_rows++;
    int len = taos_print_row(temp, row, fields, num_fields);
    len += sprintf(temp + len, "\n");
    //printf("query result:%s\n", temp);
    memcpy(databuf + totalLen, temp, len);
    totalLen += len;
H
hzcheng 已提交
1165 1166
  }

1167
  verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n",
1168 1169 1170
          __func__, __LINE__, databuf, pThreadInfo->filePath);
  if (strlen(pThreadInfo->filePath) > 0) {
    appendResultBufToFile(databuf, pThreadInfo);
1171
  }
1172 1173
  free(databuf);
}
H
hzcheng 已提交
1174

1175
static void selectAndGetResult(
1176
        threadInfo *pThreadInfo, char *command)
1177 1178 1179 1180 1181 1182 1183 1184 1185 1186
{
  if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) {
    TAOS_RES *res = taos_query(pThreadInfo->taos, command);
    if (res == NULL || taos_errno(res) != 0) {
        errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n",
            __func__, __LINE__, command, taos_errstr(res));
        taos_free_result(res);
        return;
    }

1187
    fetchResult(res, pThreadInfo);
1188
    taos_free_result(res);
1189

1190 1191 1192 1193
  } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
      int retCode = postProceSql(
              g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port,
              command,
1194
              pThreadInfo);
1195 1196 1197 1198 1199 1200 1201 1202
      if (0 != retCode) {
        printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID);
      }

  } else {
      errorPrint("%s() LN%d, unknown query mode: %s\n",
        __func__, __LINE__, g_queryInfo.queryMode);
  }
1203
}
H
hzcheng 已提交
1204

1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
static int32_t rand_bool(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 2;
}

static int32_t rand_tinyint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 128;
}

static int32_t rand_smallint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 32767;
}

static int32_t rand_int(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor];
}

static int64_t rand_bigint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randbigint[cursor];
}

static float rand_float(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
1244
  return randfloat[cursor];
1245 1246
}

1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263
#if 0
static const char charNum[] = "0123456789";

static void nonrand_string(char *, int) __attribute__ ((unused));   // reserve for debugging purpose
static void nonrand_string(char *str, int size)
{
  str[0] = 0;
  if (size > 0) {
    int n;
    for (n = 0; n < size; n++) {
      str[n] = charNum[n % 10];
    }
    str[n] = 0;
  }
}
#endif

1264
static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1265

1266
static void rand_string(char *str, int size) {
1267 1268 1269 1270
  str[0] = 0;
  if (size > 0) {
    //--size;
    int n;
1271
    for (n = 0; n < size; n++) {
1272
      int key = abs(rand_tinyint()) % (int)(sizeof(charset) - 1);
1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
      str[n] = charset[key];
    }
    str[n] = 0;
  }
}

static double rand_double() {
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randdouble[cursor];

}

static void init_rand_data() {
  for (int i = 0; i < MAX_PREPARED_RAND; i++){
1289 1290 1291 1292
    randint[i] = (int)(taosRandom() % 65535);
    randbigint[i] = (int64_t)(taosRandom() % 2147483648);
    randfloat[i] = (float)(taosRandom() / 1000.0);
    randdouble[i] = (double)(taosRandom() / 1000000.0);
1293 1294 1295
  }
}

1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1316
static int printfInsertMeta() {
1317 1318
    SHOW_PARSE_RESULT_START();

1319 1320
  printf("host:                       \033[33m%s:%u\033[0m\n",
          g_Dbs.host, g_Dbs.port);
1321 1322
  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
1323
  printf("configDir:                  \033[33m%s\033[0m\n", configDir);
1324 1325
  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
1326 1327
  printf("thread num of create table: \033[33m%d\033[0m\n",
          g_Dbs.threadCountByCreateTbl);
1328
  printf("top insert interval:        \033[33m%"PRIu64"\033[0m\n",
1329
          g_args.insert_interval);
1330
  printf("number of records per req:  \033[33m%"PRIu64"\033[0m\n",
1331
          g_args.num_of_RPR);
1332
  printf("max sql length:             \033[33m%"PRIu64"\033[0m\n",
1333
          g_args.max_sql_len);
1334 1335

  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
1336

1337 1338
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    printf("database[\033[33m%d\033[0m]:\n", i);
1339 1340
    printf("  database[%d] name:      \033[33m%s\033[0m\n",
            i, g_Dbs.db[i].dbName);
1341
    if (0 == g_Dbs.db[i].drop) {
1342 1343 1344
      printf("  drop:                  \033[33mno\033[0m\n");
    } else {
      printf("  drop:                  \033[33myes\033[0m\n");
1345 1346 1347
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
1348 1349
      printf("  blocks:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.blocks);
1350 1351
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
1352 1353
      printf("  cache:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.cache);
1354 1355
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
1356 1357
      printf("  days:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.days);
1358 1359
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
1360 1361
      printf("  keep:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.keep);
1362 1363
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
1364 1365
      printf("  replica:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.replica);
1366 1367
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
1368 1369
      printf("  update:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.update);
1370 1371
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
1372 1373
      printf("  minRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.minRows);
1374 1375
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
1376 1377
      printf("  maxRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.maxRows);
1378 1379 1380 1381 1382
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
1383 1384
      printf("  walLevel:              \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.walLevel);
1385 1386
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
1387 1388
      printf("  fsync:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.fsync);
1389 1390
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
1391 1392
      printf("  quorum:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.quorum);
1393 1394
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1395 1396
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1397 1398
        printf("  precision:             \033[33m%s\033[0m\n",
            g_Dbs.db[i].dbCfg.precision);
1399
      } else {
1400 1401 1402
        printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
                g_Dbs.db[i].dbCfg.precision);
        return -1;
1403 1404 1405
      }
    }

1406
    printf("  super table count:     \033[33m%"PRIu64"\033[0m\n",
1407
        g_Dbs.db[i].superTblCount);
1408 1409
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      printf("  super table[\033[33m%"PRIu64"\033[0m]:\n", j);
1410

1411 1412
      printf("      stbName:           \033[33m%s\033[0m\n",
          g_Dbs.db[i].superTbls[j].sTblName);
1413 1414 1415 1416 1417 1418 1419 1420

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
      }
1421

1422 1423 1424 1425 1426 1427 1428
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
      }
1429

1430
      printf("      childTblCount:     \033[33m%"PRId64"\033[0m\n",
1431 1432 1433 1434 1435 1436 1437
              g_Dbs.db[i].superTbls[j].childTblCount);
      printf("      childTblPrefix:    \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      printf("      dataSource:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      printf("      insertMode:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].insertMode);
1438
      if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
1439
        printf("      childTblLimit:     \033[33m%"PRId64"\033[0m\n",
1440
                g_Dbs.db[i].superTbls[j].childTblLimit);
1441
      }
1442
      if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) {
1443
        printf("      childTblOffset:    \033[33m%"PRIu64"\033[0m\n",
1444
                g_Dbs.db[i].superTbls[j].childTblOffset);
1445
      }
1446
      printf("      insertRows:        \033[33m%"PRId64"\033[0m\n",
1447
              g_Dbs.db[i].superTbls[j].insertRows);
1448
/*
1449
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1450
        printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
1451
      }else {
1452
        printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
1453
      }
1454
      */
1455
      printf("      interlaceRows:     \033[33m%"PRIu64"\033[0m\n",
1456
              g_Dbs.db[i].superTbls[j].interlaceRows);
1457 1458

      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1459
        printf("      stable insert interval:   \033[33m%"PRIu64"\033[0m\n",
1460 1461 1462
            g_Dbs.db[i].superTbls[j].insertInterval);
      }

1463 1464 1465 1466
      printf("      disorderRange:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      printf("      disorderRatio:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
1467
      printf("      maxSqlLen:         \033[33m%"PRIu64"\033[0m\n",
1468
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1469
      printf("      timeStampStep:     \033[33m%"PRId64"\033[0m\n",
1470 1471 1472 1473 1474 1475 1476 1477 1478
              g_Dbs.db[i].superTbls[j].timeStampStep);
      printf("      startTimestamp:    \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
      printf("      sampleFormat:      \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      printf("      sampleFile:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      printf("      tagsFile:          \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
1479
      printf("      columnCount:       \033[33m%d\033[0m\n",
1480
              g_Dbs.db[i].superTbls[j].columnCount);
1481 1482
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1483 1484 1485 1486
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "nchar", 5))) {
1487
          printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
1488 1489
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1490
        } else {
1491 1492
          printf("column[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType);
1493 1494 1495
        }
      }
      printf("\n");
1496 1497 1498

      printf("      tagCount:            \033[33m%d\033[0m\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1499 1500
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1501 1502 1503 1504
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1505
          printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
1506 1507
                  g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1508
        } else {
1509 1510
          printf("tag[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].tags[k].dataType);
1511
        }
1512 1513 1514 1515 1516
      }
      printf("\n");
    }
    printf("\n");
  }
1517 1518

  SHOW_PARSE_RESULT_END();
1519 1520

  return 0;
1521 1522 1523
}

static void printfInsertMetaToFile(FILE* fp) {
1524 1525

  SHOW_PARSE_RESULT_START_TO_FILE(fp);
1526

1527 1528
  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
  fprintf(fp, "user:                       %s\n", g_Dbs.user);
1529
  fprintf(fp, "configDir:                  %s\n", configDir);
1530 1531 1532
  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
1533 1534
  fprintf(fp, "number of records per req:  %"PRIu64"\n", g_args.num_of_RPR);
  fprintf(fp, "max sql length:             %"PRIu64"\n", g_args.max_sql_len);
1535
  fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
1536

1537 1538
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    fprintf(fp, "database[%d]:\n", i);
1539
    fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
1540
    if (0 == g_Dbs.db[i].drop) {
1541
      fprintf(fp, "  drop:                  no\n");
1542
    }else {
1543
      fprintf(fp, "  drop:                  yes\n");
1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1583 1584
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1585 1586
        fprintf(fp, "  precision:             %s\n",
                g_Dbs.db[i].dbCfg.precision);
1587
      } else {
1588 1589
        fprintf(fp, "  precision error:       %s\n",
                g_Dbs.db[i].dbCfg.precision);
1590 1591 1592
      }
    }

1593 1594
    fprintf(fp, "  super table count:     %"PRIu64"\n",
            g_Dbs.db[i].superTblCount);
1595 1596
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      fprintf(fp, "  super table[%d]:\n", j);
1597

1598 1599
      fprintf(fp, "      stbName:           %s\n",
              g_Dbs.db[i].superTbls[j].sTblName);
1600 1601 1602

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "no");
1603 1604
      } else if (AUTO_CREATE_SUBTBL
              == g_Dbs.db[i].superTbls[j].autoCreateTable) {
1605 1606 1607 1608
        fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
      } else {
        fprintf(fp, "      autoCreateTable:   %s\n",  "error");
      }
1609

1610 1611
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "no");
1612 1613
      } else if (TBL_ALREADY_EXISTS
              == g_Dbs.db[i].superTbls[j].childTblExists) {
1614 1615 1616 1617
        fprintf(fp, "      childTblExists:    %s\n",  "yes");
      } else {
        fprintf(fp, "      childTblExists:    %s\n",  "error");
      }
1618

1619
      fprintf(fp, "      childTblCount:     %"PRId64"\n",
1620 1621 1622 1623 1624 1625 1626
              g_Dbs.db[i].superTbls[j].childTblCount);
      fprintf(fp, "      childTblPrefix:    %s\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      fprintf(fp, "      dataSource:        %s\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      fprintf(fp, "      insertMode:        %s\n",
              g_Dbs.db[i].superTbls[j].insertMode);
1627
      fprintf(fp, "      insertRows:        %"PRId64"\n",
1628
              g_Dbs.db[i].superTbls[j].insertRows);
1629
      fprintf(fp, "      interlace rows:    %"PRIu64"\n",
1630
              g_Dbs.db[i].superTbls[j].interlaceRows);
1631
      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1632
        fprintf(fp, "      stable insert interval:   %"PRIu64"\n",
1633
                g_Dbs.db[i].superTbls[j].insertInterval);
1634
      }
1635
/*
1636
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1637
        fprintf(fp, "      multiThreadWriteOneTbl:  no\n");
1638
      }else {
1639
        fprintf(fp, "      multiThreadWriteOneTbl:  yes\n");
1640
      }
1641
      */
1642
      fprintf(fp, "      interlaceRows:     %"PRIu64"\n",
1643
              g_Dbs.db[i].superTbls[j].interlaceRows);
1644 1645 1646 1647
      fprintf(fp, "      disorderRange:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      fprintf(fp, "      disorderRatio:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
1648
      fprintf(fp, "      maxSqlLen:         %"PRIu64"\n",
1649
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1650

1651 1652 1653 1654
      fprintf(fp, "      timeStampStep:     %"PRId64"\n",
              g_Dbs.db[i].superTbls[j].timeStampStep);
      fprintf(fp, "      startTimestamp:    %s\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
1655 1656 1657 1658 1659 1660
      fprintf(fp, "      sampleFormat:      %s\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      fprintf(fp, "      sampleFile:        %s\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      fprintf(fp, "      tagsFile:          %s\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
1661

1662 1663
      fprintf(fp, "      columnCount:       %d\n        ",
              g_Dbs.db[i].superTbls[j].columnCount);
1664 1665
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1666 1667 1668
        if ((0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "binary", strlen("binary")))
1669 1670
                || (0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
1671 1672 1673 1674
                        "nchar", strlen("nchar")))) {
          fprintf(fp, "column[%d]:%s(%d) ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1675
        } else {
1676 1677
          fprintf(fp, "column[%d]:%s ",
                  k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
1678 1679 1680
        }
      }
      fprintf(fp, "\n");
1681 1682 1683

      fprintf(fp, "      tagCount:            %d\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1684 1685
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1686 1687 1688 1689
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1690 1691
          fprintf(fp, "tag[%d]:%s(%d) ",
                  k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
1692
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1693 1694
        } else {
          fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
1695
        }
1696 1697 1698 1699 1700
      }
      fprintf(fp, "\n");
    }
    fprintf(fp, "\n");
  }
1701

1702
  SHOW_PARSE_RESULT_END_TO_FILE(fp);
1703 1704 1705
}

static void printfQueryMeta() {
1706

1707
  SHOW_PARSE_RESULT_START();
1708

1709 1710
  printf("host:                    \033[33m%s:%u\033[0m\n",
          g_queryInfo.host, g_queryInfo.port);
1711 1712 1713 1714
  printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
  printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

  printf("\n");
1715 1716 1717 1718

  if ((SUBSCRIBE_TEST == g_args.test_mode) || (QUERY_TEST == g_args.test_mode)) {
    printf("specified table query info:                   \n");
    printf("sqlCount:       \033[33m%"PRIu64"\033[0m\n",
1719
      g_queryInfo.specifiedQueryInfo.sqlCount);
1720 1721 1722
    if (g_queryInfo.specifiedQueryInfo.sqlCount > 0) {
      printf("specified tbl query times:\n");
      printf("                \033[33m%"PRIu64"\033[0m\n",
1723
      g_queryInfo.specifiedQueryInfo.queryTimes);
1724 1725 1726
      printf("query interval: \033[33m%"PRIu64" ms\033[0m\n",
        g_queryInfo.specifiedQueryInfo.queryInterval);
      printf("top query times:\033[33m%"PRIu64"\033[0m\n", g_args.query_times);
1727
      printf("concurrent:     \033[33m%d\033[0m\n",
1728 1729
      g_queryInfo.specifiedQueryInfo.concurrent);
      printf("mod:            \033[33m%s\033[0m\n",
1730
        (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync");
1731
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1732
        g_queryInfo.specifiedQueryInfo.subscribeInterval);
1733
      printf("restart:        \033[33m%d\033[0m\n",
1734
        g_queryInfo.specifiedQueryInfo.subscribeRestart);
1735
      printf("keepProgress:   \033[33m%d\033[0m\n",
1736
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
1737

1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753
      for (uint64_t i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        printf("  sql[%"PRIu64"]: \033[33m%s\033[0m\n",
            i, g_queryInfo.specifiedQueryInfo.sql[i]);
      }
      printf("\n");
    }

    printf("super table query info:\n");
    printf("sqlCount:       \033[33m%"PRIu64"\033[0m\n",
      g_queryInfo.superQueryInfo.sqlCount);

    if (g_queryInfo.superQueryInfo.sqlCount > 0) {
      printf("query interval: \033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryInterval);
      printf("threadCnt:      \033[33m%d\033[0m\n",
        g_queryInfo.superQueryInfo.threadCnt);
1754
      printf("childTblCount:  \033[33m%"PRId64"\033[0m\n",
1755 1756 1757 1758 1759 1760 1761
        g_queryInfo.superQueryInfo.childTblCount);
      printf("stable name:    \033[33m%s\033[0m\n",
        g_queryInfo.superQueryInfo.sTblName);
      printf("stb query times:\033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryTimes);

      printf("mod:            \033[33m%s\033[0m\n",
1762
        (g_queryInfo.superQueryInfo.asyncMode)?"async":"sync");
1763
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1764
        g_queryInfo.superQueryInfo.subscribeInterval);
1765
      printf("restart:        \033[33m%d\033[0m\n",
1766
        g_queryInfo.superQueryInfo.subscribeRestart);
1767
      printf("keepProgress:   \033[33m%d\033[0m\n",
1768 1769
        g_queryInfo.superQueryInfo.subscribeKeepProgress);

1770 1771 1772 1773 1774 1775
      for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
        printf("  sql[%d]: \033[33m%s\033[0m\n",
            i, g_queryInfo.superQueryInfo.sql[i]);
      }
      printf("\n");
    }
1776
  }
1777

1778
  SHOW_PARSE_RESULT_END();
1779 1780
}

1781
static char* formatTimestamp(char* buf, int64_t val, int precision) {
1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812
  time_t tt;
  if (precision == TSDB_TIME_PRECISION_MICRO) {
    tt = (time_t)(val / 1000000);
  } else {
    tt = (time_t)(val / 1000);
  }

/* comment out as it make testcases like select_with_tags.sim fail.
  but in windows, this may cause the call to localtime crash if tt < 0,
  need to find a better solution.
  if (tt < 0) {
    tt = 0;
  }
  */

#ifdef WINDOWS
  if (tt < 0) tt = 0;
#endif

  struct tm* ptm = localtime(&tt);
  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);

  if (precision == TSDB_TIME_PRECISION_MICRO) {
    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
  } else {
    sprintf(buf + pos, ".%03d", (int)(val % 1000));
  }

  return buf;
}

1813 1814 1815
static void xDumpFieldToFile(FILE* fp, const char* val,
        TAOS_FIELD* field, int32_t length, int precision) {

1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850
  if (val == NULL) {
    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
    return;
  }

  char buf[TSDB_MAX_BYTES_PER_ROW];
  switch (field->type) {
    case TSDB_DATA_TYPE_BOOL:
      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
      break;
    case TSDB_DATA_TYPE_TINYINT:
      fprintf(fp, "%d", *((int8_t *)val));
      break;
    case TSDB_DATA_TYPE_SMALLINT:
      fprintf(fp, "%d", *((int16_t *)val));
      break;
    case TSDB_DATA_TYPE_INT:
      fprintf(fp, "%d", *((int32_t *)val));
      break;
    case TSDB_DATA_TYPE_BIGINT:
      fprintf(fp, "%" PRId64, *((int64_t *)val));
      break;
    case TSDB_DATA_TYPE_FLOAT:
      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
      break;
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      memcpy(buf, val, length);
      buf[length] = 0;
      fprintf(fp, "\'%s\'", buf);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
1851
      formatTimestamp(buf, *(int64_t*)val, precision);
1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866
      fprintf(fp, "'%s'", buf);
      break;
    default:
      break;
  }
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  FILE* fp = fopen(fname, "at");
  if (fp == NULL) {
1867
    errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881
    return -1;
  }

  int num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int precision = taos_result_precision(tres);

  for (int col = 0; col < num_fields; col++) {
    if (col > 0) {
      fprintf(fp, ",");
    }
    fprintf(fp, "%s", fields[col].name);
  }
  fputc('\n', fp);
1882

1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902
  int numOfRows = 0;
  do {
    int32_t* length = taos_fetch_lengths(tres);
    for (int i = 0; i < num_fields; i++) {
      if (i > 0) {
        fputc(',', fp);
      }
      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
    }
    fputc('\n', fp);

    numOfRows++;
    row = taos_fetch_row(tres);
  } while( row != NULL);

  fclose(fp);

  return numOfRows;
}

1903 1904
static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
  TAOS_RES * res;
1905 1906
  TAOS_ROW row = NULL;
  int count = 0;
1907 1908

  res = taos_query(taos, "show databases;");
1909
  int32_t code = taos_errno(res);
1910

1911
  if (code != 0) {
1912
    errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
1913 1914 1915 1916 1917
    return -1;
  }

  TAOS_FIELD *fields = taos_fetch_fields(res);

1918
  while((row = taos_fetch_row(res)) != NULL) {
1919
    // sys database name : 'log'
1920 1921 1922 1923
    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log",
                fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) {
      continue;
    }
1924 1925 1926

    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
    if (dbInfos[count] == NULL) {
1927
      errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
1928 1929 1930
      return -1;
    }

1931 1932
    tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
            fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
1933
    formatTimestamp(dbInfos[count]->create_time,
1934 1935
            *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
            TSDB_TIME_PRECISION_MILLI);
1936
    dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
1937
    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
1938 1939
    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
1940
    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
1941

1942
    tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
1943
            fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
1944 1945 1946 1947 1948 1949 1950
    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
1951 1952
    dbInfos[count]->cachelast =
      (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));
1953

1954
    tstrncpy(dbInfos[count]->precision,
1955
            (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
1956
            fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
1957
    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
1958
    tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
1959
            fields[TSDB_SHOW_DB_STATUS_INDEX].bytes);
1960

1961 1962
    count++;
    if (count > MAX_DATABASE_COUNT) {
1963 1964
      errorPrint("%s() LN%d, The database count overflow than %d\n",
         __func__, __LINE__, MAX_DATABASE_COUNT);
1965 1966 1967 1968 1969 1970 1971
      break;
    }
  }

  return count;
}

1972 1973
static void printfDbInfoForQueryToFile(
        char* filename, SDbInfo* dbInfos, int index) {
1974

1975
  if (filename[0] == 0)
1976
      return;
1977 1978 1979

  FILE *fp = fopen(filename, "at");
  if (fp == NULL) {
1980
    errorPrint( "failed to open file: %s\n", filename);
1981
    return;
1982 1983 1984 1985 1986
  }

  fprintf(fp, "================ database[%d] ================\n", index);
  fprintf(fp, "name: %s\n", dbInfos->name);
  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
1987
  fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables);
1988
  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);
1989 1990
  fprintf(fp, "replica: %d\n", dbInfos->replica);
  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
1991 1992
  fprintf(fp, "days: %d\n", dbInfos->days);
  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);
1993 1994 1995 1996 1997 1998 1999
  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
  fprintf(fp, "comp: %d\n", dbInfos->comp);
2000 2001
  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);
  fprintf(fp, "precision: %s\n", dbInfos->precision);
2002
  fprintf(fp, "update: %d\n", dbInfos->update);
2003
  fprintf(fp, "status: %s\n", dbInfos->status);
2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017
  fprintf(fp, "\n");

  fclose(fp);
}

static void printfQuerySystemInfo(TAOS * taos) {
  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
  TAOS_RES* res;

  time_t t;
  struct tm* lt;
  time(&t);
  lt = localtime(&t);
2018 2019 2020 2021
  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
          lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
          lt->tm_sec);

2022 2023
  // show variables
  res = taos_query(taos, "show variables;");
2024
  //fetchResult(res, filename);
2025 2026 2027 2028 2029
  xDumpResultToFile(filename, res);

  // show dnodes
  res = taos_query(taos, "show dnodes;");
  xDumpResultToFile(filename, res);
2030
  //fetchResult(res, filename);
2031

2032 2033 2034 2035
  // show databases
  res = taos_query(taos, "show databases;");
  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
  if (dbInfos == NULL) {
2036
    errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
2037 2038 2039
    return;
  }
  int dbCount = getDbFromServer(taos, dbInfos);
2040 2041 2042 2043
  if (dbCount <= 0) {
      free(dbInfos);
      return;
  }
2044 2045

  for (int i = 0; i < dbCount; i++) {
2046
    // printf database info
2047
    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
2048

2049 2050 2051 2052
    // show db.vgroups
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);
2053

2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064
    // show db.stables
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);

    free(dbInfos[i]);
  }

  free(dbInfos);
}

2065
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
2066
        char* sqlstr, threadInfo *pThreadInfo)
2067
{
2068
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
2069 2070 2071

    char *url = "/rest/sql";

2072
    int bytes, sent, received, req_str_len, resp_len;
2073 2074 2075
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
2076

2077 2078 2079
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;

    request_buf = malloc(req_buf_len);
2080 2081 2082 2083
    if (NULL == request_buf) {
      errorPrint("%s", "ERROR, cannot allocate memory.\n");
      exit(EXIT_FAILURE);
    }
2084

2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};

    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
2096

2097 2098 2099 2100 2101 2102
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);

    char base64_buf[INPUT_BUF_LEN];
2103
#ifdef WINDOWS
2104
    WSADATA wsaData;
2105
    WSAStartup(MAKEWORD(2, 2), &wsaData);
2106 2107 2108 2109
    SOCKET sockfd;
#else
    int sockfd;
#endif
2110
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
2111
    if (sockfd < 0) {
2112
#ifdef WINDOWS
2113
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
2114
#endif
2115
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
2116
        free(request_buf);
2117
        ERROR_EXIT("ERROR opening socket");
2118
    }
2119

2120
    int retConn = connect(sockfd, (struct sockaddr *)pServAddr, sizeof(struct sockaddr));
2121 2122
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
2123
        free(request_buf);
2124
        ERROR_EXIT("ERROR connecting");
2125
    }
2126

2127 2128 2129
    memset(base64_buf, 0, INPUT_BUF_LEN);

    for (int n = 0, m = 0; n < userpass_buf_len;) {
2130
      uint32_t oct_a = n < userpass_buf_len ?
2131
        (unsigned char) userpass_buf[n++]:0;
2132
      uint32_t oct_b = n < userpass_buf_len ?
2133
        (unsigned char) userpass_buf[n++]:0;
2134
      uint32_t oct_c = n < userpass_buf_len ?
2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;

      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }

    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';

2147 2148
    debugPrint("%s() LN%d: auth string base64 encoded: %s\n",
            __func__, __LINE__, base64_buf);
2149 2150
    char *auth = base64_buf;

2151 2152 2153
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
2154 2155 2156 2157 2158
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
2159
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
2160

2161
    req_str_len = strlen(request_buf);
2162 2163
    sent = 0;
    do {
2164 2165 2166
#ifdef WINDOWS
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
#else
2167
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
2168
#endif
2169
        if (bytes < 0)
2170
            ERROR_EXIT("ERROR writing message to socket");
2171 2172 2173
        if (bytes == 0)
            break;
        sent+=bytes;
2174
    } while(sent < req_str_len);
2175

2176 2177
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
2178 2179
    received = 0;
    do {
2180 2181 2182
#ifdef WINDOWS
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
#else
2183
        bytes = read(sockfd, response_buf + received, resp_len - received);
2184
#endif
2185 2186
        if (bytes < 0) {
            free(request_buf);
2187
            ERROR_EXIT("ERROR reading response from socket");
2188
        }
2189 2190
        if (bytes == 0)
            break;
2191
        received += bytes;
2192
    } while(received < resp_len);
2193

2194 2195
    if (received == resp_len) {
        free(request_buf);
2196
        ERROR_EXIT("ERROR storing complete response from socket");
2197
    }
2198

2199
    response_buf[RESP_BUF_LEN - 1] = '\0';
2200 2201
    printf("Response:\n%s\n", response_buf);

2202 2203
    if (strlen(pThreadInfo->filePath) > 0) {
       appendResultBufToFile(response_buf, pThreadInfo);
2204 2205
    }

2206
    free(request_buf);
2207 2208 2209 2210
#ifdef WINDOWS
    closesocket(sockfd);
    WSACleanup();
#else
2211
    close(sockfd);
2212
#endif
2213 2214 2215 2216

    return 0;
}

2217
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
2218 2219
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
2220 2221
    errorPrint("%s() LN%d, calloc failed! size:%d\n",
        __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
2222 2223
    return NULL;
  }
2224

2225
  int    dataLen = 0;
2226 2227 2228
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);

2229 2230 2231
  return dataBuf;
}

2232
static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) {
2233 2234 2235 2236 2237 2238 2239 2240 2241
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
    printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
    return NULL;
  }

  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
  for (int i = 0; i < stbInfo->tagCount; i++) {
2242 2243
    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
2244
      if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
2245 2246
        printf("binary or nchar length overflow, max size:%u\n",
                (uint32_t)TSDB_MAX_BINARY_LEN);
2247 2248 2249
        tmfree(dataBuf);
        return NULL;
      }
2250

2251 2252
      int tagBufLen = stbInfo->tags[i].dataLen + 1;
      char* buf = (char*)calloc(tagBufLen, 1);
2253 2254 2255 2256 2257
      if (NULL == buf) {
        printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
        tmfree(dataBuf);
        return NULL;
      }
2258 2259 2260 2261 2262 2263 2264

      if (tableSeq % 2) {
        tstrncpy(buf, "beijing", tagBufLen);
      } else {
        tstrncpy(buf, "shanghai", tagBufLen);
      }
      //rand_string(buf, stbInfo->tags[i].dataLen);
2265 2266
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "\'%s\', ", buf);
2267
      tmfree(buf);
2268 2269 2270
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "int", strlen("int"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2271
              "%d, ", tableSeq);
2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bigint", strlen("bigint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "float", strlen("float"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_float());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "double", strlen("double"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_double());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "smallint", strlen("smallint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_smallint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "tinyint", strlen("tinyint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_tinyint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bool", strlen("bool"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_bool());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "timestamp", strlen("timestamp"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
2300 2301 2302 2303 2304 2305
    }  else {
      printf("No support data type: %s\n", stbInfo->tags[i].dataType);
      tmfree(dataBuf);
      return NULL;
    }
  }
2306

2307
  dataLen -= 2;
2308
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");
2309 2310 2311
  return dataBuf;
}

2312
static int calcRowLen(SSuperTable*  superTbls) {
2313 2314
  int colIndex;
  int  lenOfOneRow = 0;
2315

2316 2317
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2318

2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
2335
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2336
      lenOfOneRow += 42;
2337
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
2351

2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
2368
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2369 2370 2371 2372 2373 2374 2375 2376
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2377

2378 2379 2380 2381
  return 0;
}


2382 2383
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
2384
        int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) {
2385

2386
  char command[BUFFER_SIZE] = "\0";
2387 2388
  char limitBuf[100] = "\0";

2389
  TAOS_RES * res;
2390 2391 2392
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2393 2394

  if (offset >= 0) {
2395
    snprintf(limitBuf, 100, " limit %"PRId64" offset %"PRIu64"",
2396
            limit, offset);
2397 2398
  }

2399
  //get all child table name use cmd: select tbname from superTblName;
2400 2401
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s",
          dbName, sTblName, limitBuf);
2402

2403
  res = taos_query(taos, command);
2404 2405 2406 2407
  int32_t code = taos_errno(res);
  if (code != 0) {
    taos_free_result(res);
    taos_close(taos);
2408 2409
    errorPrint("%s() LN%d, failed to run command %s\n",
           __func__, __LINE__, command);
2410 2411 2412
    exit(-1);
  }

2413 2414
  int64_t childTblCount = (limit < 0)?10000:limit;
  int64_t count = 0;
2415 2416 2417 2418 2419 2420 2421 2422 2423 2424
  if (childTblName == NULL) {
    childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (NULL ==  childTblName) {
    taos_free_result(res);
        taos_close(taos);
        errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__);
        exit(-1);
    }
  }

2425
  char* pTblName = childTblName;
2426
  while((row = taos_fetch_row(res)) != NULL) {
2427
    int32_t* len = taos_fetch_lengths(res);
2428
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2429 2430 2431
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2432 2433
      char *tmp = realloc(childTblName,
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2434 2435 2436
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2437 2438
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2439 2440
      } else {
        // exit, if allocate more memory failed
2441 2442
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
               __func__, __LINE__, dbName, sTblName);
2443 2444 2445 2446 2447 2448
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2449
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2450
  }
2451

2452 2453 2454 2455 2456 2457 2458
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2459 2460
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
        char* sTblName, char** childTblNameOfSuperTbl,
2461
        int64_t* childTblCountOfSuperTbl) {
2462 2463 2464

    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
2465
            -1, 0);
2466 2467
}

2468 2469
static int getSuperTableFromServer(TAOS * taos, char* dbName,
        SSuperTable*  superTbls) {
2470

2471
  char command[BUFFER_SIZE] = "\0";
2472
  TAOS_RES * res;
2473 2474
  TAOS_ROW row = NULL;
  int count = 0;
2475

2476
  //get schema use cmd: describe superTblName;
2477
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2478
  res = taos_query(taos, command);
2479 2480 2481
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2482
    taos_free_result(res);
2483 2484 2485 2486 2487 2488
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
2489
  while((row = taos_fetch_row(res)) != NULL) {
2490 2491 2492
    if (0 == count) {
      count++;
      continue;
2493
    }
2494 2495

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506
      tstrncpy(superTbls->tags[tagIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->tags[tagIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2507
      tagIndex++;
2508
    } else {
2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519
      tstrncpy(superTbls->columns[columnIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->columns[columnIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2531
/*
2532
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
2533
    //get all child table name use cmd: select tbname from superTblName;
2534 2535 2536
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2537
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2538 2539
      return -1;
    }
2540
    getAllChildNameOfSuperTable(taos, dbName,
2541 2542 2543
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2544
  }
2545
  */
2546 2547 2548
  return 0;
}

2549 2550
static int createSuperTable(
        TAOS * taos, char* dbName,
2551
        SSuperTable*  superTbl) {
2552

2553
  char command[BUFFER_SIZE] = "\0";
2554

2555 2556 2557 2558 2559
  char cols[STRING_LEN] = "\0";
  int colIndex;
  int len = 0;

  int  lenOfOneRow = 0;
2560 2561 2562 2563 2564 2565 2566 2567 2568

  if (superTbl->columnCount == 0) {
    errorPrint("%s() LN%d, super table column count is %d\n",
            __func__, __LINE__, superTbl->columnCount);
    return -1;
  }

  for (colIndex = 0; colIndex < superTbl->columnCount; colIndex++) {
    char* dataType = superTbl->columns[colIndex].dataType;
2569

2570
    if (strcasecmp(dataType, "BINARY") == 0) {
2571 2572
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "BINARY",
2573 2574
          superTbl->columns[colIndex].dataLen);
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2575
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
2576 2577
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "NCHAR",
2578 2579
          superTbl->columns[colIndex].dataLen);
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
      lenOfOneRow += 22;
2598
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2599 2600
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
      lenOfOneRow += 42;
2601
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2602 2603 2604 2605
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
      lenOfOneRow += 21;
    } else {
      taos_close(taos);
2606 2607
      errorPrint("%s() LN%d, config error data type : %s\n",
         __func__, __LINE__, dataType);
2608 2609 2610 2611
      exit(-1);
    }
  }

2612 2613
  superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp
  //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbl[j].sTblName, g_Dbs.db[i].superTbl[j].columnCount, lenOfOneRow);
2614 2615

  // save for creating child table
2616 2617 2618 2619
  superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1);
  if (NULL == superTbl->colsOfCreateChildTable) {
    errorPrint("%s() LN%d, Failed when calloc, size:%d",
           __func__, __LINE__, len+1);
2620 2621 2622 2623
    taos_close(taos);
    exit(-1);
  }

2624
  snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
2625 2626
  verbosePrint("%s() LN%d: %s\n",
      __func__, __LINE__, superTbl->colsOfCreateChildTable);
2627

2628 2629 2630 2631 2632
  if (superTbl->tagCount == 0) {
    errorPrint("%s() LN%d, super table tag count is %d\n",
            __func__, __LINE__, superTbl->tagCount);
    return -1;
  }
2633

2634 2635 2636
  char tags[STRING_LEN] = "\0";
  int tagIndex;
  len = 0;
2637

2638 2639 2640 2641
  int lenOfTagOfOneRow = 0;
  len += snprintf(tags + len, STRING_LEN - len, "(");
  for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) {
    char* dataType = superTbl->tags[tagIndex].dataType;
2642

2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680
    if (strcasecmp(dataType, "BINARY") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
              "BINARY", superTbl->tags[tagIndex].dataLen);
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
              "NCHAR", superTbl->tags[tagIndex].dataLen);
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "INT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "BIGINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "SMALLINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "TINYINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "BOOL");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "FLOAT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "DOUBLE");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42;
    } else {
      taos_close(taos);
2681 2682
      errorPrint("%s() LN%d, config error tag type : %s\n",
         __func__, __LINE__, dataType);
2683
      exit(-1);
2684 2685
    }
  }
2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702

  len -= 2;
  len += snprintf(tags + len, STRING_LEN - len, ")");

  superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow;

  snprintf(command, BUFFER_SIZE,
          "create table if not exists %s.%s (ts timestamp%s) tags %s",
          dbName, superTbl->sTblName, cols, tags);
  verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command);

  if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
      errorPrint( "create supertable %s failed!\n\n",
              superTbl->sTblName);
      return -1;
  }
  debugPrint("create supertable %s success!\n\n", superTbl->sTblName);
2703 2704 2705
  return 0;
}

2706
static int createDatabasesAndStables() {
2707 2708 2709 2710
  TAOS * taos = NULL;
  int    ret = 0;
  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
  if (taos == NULL) {
2711
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2712
    return -1;
2713 2714 2715
  }
  char command[BUFFER_SIZE] = "\0";

2716
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2717 2718
    if (g_Dbs.db[i].drop) {
      sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
2719
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);
2720
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2721 2722 2723 2724
        taos_close(taos);
        return -1;
      }

2725
      int dataLen = 0;
2726
      dataLen += snprintf(command + dataLen,
2727
          BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2728

2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798
      if (g_Dbs.db[i].dbCfg.blocks > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
      }
      if (g_Dbs.db[i].dbCfg.cache > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
      }
      if (g_Dbs.db[i].dbCfg.days > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
      }
      if (g_Dbs.db[i].dbCfg.keep > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
      }
      if (g_Dbs.db[i].dbCfg.quorum > 1) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
      }
      if (g_Dbs.db[i].dbCfg.replica > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
      }
      if (g_Dbs.db[i].dbCfg.update > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
      }
      //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
      //  dataLen += snprintf(command + dataLen,
      //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
      //}
      if (g_Dbs.db[i].dbCfg.minRows > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
      }
      if (g_Dbs.db[i].dbCfg.maxRows > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
      }
      if (g_Dbs.db[i].dbCfg.comp > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
      }
      if (g_Dbs.db[i].dbCfg.walLevel > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
      }
      if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
      }
      if (g_Dbs.db[i].dbCfg.fsync > 0) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
                " fsync %d", g_Dbs.db[i].dbCfg.fsync);
      }
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                      "us", strlen("us")))) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
                " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
      }

      debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
        taos_close(taos);
        errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
        return -1;
      }
      printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
2799 2800
    }

2801
    debugPrint("%s() LN%d supertbl count:%"PRIu64"\n",
2802
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
2803 2804 2805

    int validStbCount = 0;

2806
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
2807 2808 2809 2810 2811 2812 2813
      sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
              g_Dbs.db[i].superTbls[j].sTblName);
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);

      ret = queryDbExec(taos, command, NO_INSERT_TYPE, true);

      if ((ret != 0) || (g_Dbs.db[i].drop)) {
2814
        ret = createSuperTable(taos, g_Dbs.db[i].dbName,
2815
                &g_Dbs.db[i].superTbls[j]);
2816

2817
        if (0 != ret) {
2818 2819
          errorPrint("create super table %d failed!\n\n", j);
          continue;
2820 2821 2822
        }
      }

2823
      ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
2824
                &g_Dbs.db[i].superTbls[j]);
2825 2826 2827
      if (0 != ret) {
        errorPrint("\nget super table %s.%s info failed!\n\n",
                g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName);
2828
        continue;
2829
      }
2830 2831

      validStbCount ++;
2832
    }
2833 2834

    g_Dbs.db[i].superTblCount = validStbCount;
2835 2836 2837 2838 2839 2840
  }

  taos_close(taos);
  return 0;
}

2841 2842
static void* createTable(void *sarg)
{
2843 2844
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
2845 2846 2847

  int64_t  lastPrintTime = taosGetTimestampMs();

2848
  int buff_len;
2849
  buff_len = BUFFER_SIZE / 8;
2850

2851
  char *buffer = calloc(buff_len, 1);
2852
  if (buffer == NULL) {
2853
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
2854 2855
    exit(-1);
  }
2856 2857 2858

  int len = 0;
  int batchNum = 0;
2859

2860
  verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n",
2861
          __func__, __LINE__,
2862
          pThreadInfo->start_table_from, pThreadInfo->end_table_to);
2863

2864
  for (uint64_t i = pThreadInfo->start_table_from;
2865
          i <= pThreadInfo->end_table_to; i++) {
2866
    if (0 == g_Dbs.use_metric) {
2867
      snprintf(buffer, buff_len,
2868
              "create table if not exists %s.%s%"PRIu64" %s;",
2869
              pThreadInfo->db_name,
2870
              g_args.tb_prefix, i,
2871
              pThreadInfo->cols);
2872
    } else {
2873 2874 2875
      if (superTblInfo == NULL) {
        errorPrint("%s() LN%d, use metric, but super table info is NULL\n",
                  __func__, __LINE__);
2876
        free(buffer);
2877 2878 2879 2880 2881 2882 2883 2884 2885 2886
        exit(-1);
      } else {
        if (0 == len) {
          batchNum = 0;
          memset(buffer, 0, buff_len);
          len += snprintf(buffer + len,
                  buff_len - len, "create table ");
        }
        char* tagsValBuf = NULL;
        if (0 == superTblInfo->tagSource) {
2887
          tagsValBuf = generateTagVaulesForStb(superTblInfo, i);
2888 2889 2890 2891 2892 2893 2894 2895 2896 2897
        } else {
          tagsValBuf = getTagValueFromTagSample(
                  superTblInfo,
                  i % superTblInfo->tagSampleCount);
        }
        if (NULL == tagsValBuf) {
          free(buffer);
          return NULL;
        }
        len += snprintf(buffer + len,
2898
                buff_len - len,
2899
                "if not exists %s.%s%"PRIu64" using %s.%s tags %s ",
2900 2901
                pThreadInfo->db_name, superTblInfo->childTblPrefix,
                i, pThreadInfo->db_name,
2902 2903 2904 2905
                superTblInfo->sTblName, tagsValBuf);
        free(tagsValBuf);
        batchNum++;
        if ((batchNum < superTblInfo->batchCreateTableNum)
2906
                && ((buff_len - len)
2907 2908 2909
                    >= (superTblInfo->lenOfTagOfOneRow + 256))) {
          continue;
        }
2910 2911 2912 2913
      }
    }

    len = 0;
2914
    verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
2915
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)){
2916
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2917
      free(buffer);
2918 2919 2920 2921 2922
      return NULL;
    }

    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
2923
      printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n",
2924
              pThreadInfo->threadID, pThreadInfo->start_table_from, i);
2925 2926 2927
      lastPrintTime = currentPrintTime;
    }
  }
2928

2929
  if (0 != len) {
2930
    verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer);
2931
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)) {
2932
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2933
    }
2934
  }
2935

2936 2937 2938 2939
  free(buffer);
  return NULL;
}

2940
static int startMultiThreadCreateChildTable(
2941
        char* cols, int threads, uint64_t startFrom, int64_t ntables,
2942
        char* db_name, SSuperTable* superTblInfo) {
2943

2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  threadInfo *infos = malloc(threads * sizeof(threadInfo));

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

2956
  int64_t a = ntables / threads;
2957 2958 2959 2960 2961
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

2962
  int64_t b = 0;
2963
  b = ntables % threads;
2964

2965
  for (int64_t i = 0; i < threads; i++) {
2966 2967 2968 2969
    threadInfo *pThreadInfo = infos + i;
    pThreadInfo->threadID = i;
    tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE);
    pThreadInfo->superTblInfo = superTblInfo;
2970
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
2971
    pThreadInfo->taos = taos_connect(
2972 2973 2974 2975 2976
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
2977
    if (pThreadInfo->taos == NULL) {
2978 2979
      errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n",
         __func__, __LINE__, taos_errstr(NULL));
2980
      free(pids);
2981
      free(infos);
2982 2983
      return -1;
    }
2984

2985 2986 2987 2988 2989 2990 2991 2992
    pThreadInfo->start_table_from = startFrom;
    pThreadInfo->ntables = i<b?a+1:a;
    pThreadInfo->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
    startFrom = pThreadInfo->end_table_to + 1;
    pThreadInfo->use_metric = true;
    pThreadInfo->cols = cols;
    pThreadInfo->minDelay = UINT64_MAX;
    pthread_create(pids + i, NULL, createTable, pThreadInfo);
2993
  }
2994

2995 2996 2997 2998 2999
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
3000 3001
    threadInfo *pThreadInfo = infos + i;
    taos_close(pThreadInfo->taos);
3002 3003 3004
  }

  free(pids);
3005
  free(infos);
3006 3007

  return 0;
3008 3009
}

3010 3011 3012 3013
static void createChildTables() {
    char tblColsBuf[MAX_SQL_SIZE];
    int len;

3014
  for (int i = 0; i < g_Dbs.dbCount; i++) {
3015 3016 3017 3018 3019 3020 3021 3022
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
          // with super table
        for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
          if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
                || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
            continue;
          }
3023

3024 3025
          verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
                  g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
3026
          uint64_t startFrom = 0;
3027 3028
          g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;

3029
          verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n",
3030 3031 3032 3033 3034 3035 3036 3037
                  __func__, __LINE__, g_totalChildTables, startFrom);
          startMultiThreadCreateChildTable(
                g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
                g_Dbs.threadCountByCreateTbl,
                startFrom,
                g_Dbs.db[i].superTbls[j].childTblCount,
                g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
        }
3038 3039
      }
    } else {
3040 3041
      // normal table
      len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
3042
      for (int j = 0; j < g_args.num_of_CPR; j++) {
3043 3044 3045 3046
          if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
                  || (strncasecmp(g_args.datatype[j],
                      "NCHAR", strlen("NCHAR")) == 0)) {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
3047
                      ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary);
3048 3049 3050 3051 3052 3053
          } else {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                      ", COL%d %s", j, g_args.datatype[j]);
          }
          len = strlen(tblColsBuf);
      }
3054

3055
      snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");
3056

3057
      verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n",
3058 3059 3060 3061 3062 3063 3064 3065 3066
              __func__, __LINE__,
              g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
      startMultiThreadCreateChildTable(
            tblColsBuf,
            g_Dbs.threadCountByCreateTbl,
            0,
            g_args.num_of_tables,
            g_Dbs.db[i].dbName,
            NULL);
3067
    }
3068 3069 3070 3071 3072 3073
  }
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3074
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
3075 3076 3077
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
3078

3079 3080
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
3081 3082
    printf("Failed to open tags file: %s, reason:%s\n",
            superTblInfo->tagsFile, strerror(errno));
3083 3084 3085 3086 3087 3088 3089
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
3090

3091 3092 3093 3094 3095 3096 3097 3098 3099
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

3100
  while((readLen = tgetline(&line, &n, fp)) != -1) {
3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
3113 3114
      char *tmp = realloc(tagDataBuf,
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
3115 3116 3117
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
3118 3119
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

int readSampleFromJsonFileToMem(SSuperTable  * superTblInfo) {
  // TODO
  return 0;
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3147 3148
static int readSampleFromCsvFileToMem(
        SSuperTable* superTblInfo) {
3149 3150 3151 3152
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
3153

3154 3155
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
3156
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
3157 3158 3159
              superTblInfo->sampleFile, strerror(errno));
      return -1;
  }
3160

3161
  assert(superTblInfo->sampleDataBuf);
3162 3163
  memset(superTblInfo->sampleDataBuf, 0,
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
3164
  while(1) {
3165
    readLen = tgetline(&line, &n, fp);
3166 3167
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
3168
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
3169
                superTblInfo->sampleFile, strerror(errno));
3170
        fclose(fp);
3171 3172 3173 3174
        return -1;
      }
      continue;
    }
3175

3176 3177 3178 3179 3180 3181 3182 3183 3184
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
3185
      printf("sample row len[%d] overflow define schema len[%"PRIu64"], so discard this row\n",
3186
              (int32_t)readLen, superTblInfo->lenOfOneRow);
3187 3188 3189
      continue;
    }

3190 3191
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
          line, readLen);
3192 3193 3194 3195 3196 3197 3198
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

3199
  fclose(fp);
3200 3201 3202 3203
  tmfree(line);
  return 0;
}

3204 3205
static bool getColumnAndTagTypeFromInsertJsonFile(
        cJSON* stbInfo, SSuperTable* superTbls) {
3206
  bool  ret = false;
3207

3208
  // columns
3209 3210
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
3211
    printf("ERROR: failed to read json, columns not found\n");
3212 3213 3214 3215 3216 3217
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
3218

3219
  int columnSize = cJSON_GetArraySize(columns);
3220
  if ((columnSize + 1/* ts */) > MAX_COLUMN_COUNT) {
3221 3222
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
            __func__, __LINE__, MAX_COLUMN_COUNT);
3223 3224 3225 3226 3227 3228
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
3229

3230
  //superTbls->columnCount = columnSize;
3231 3232 3233 3234 3235 3236 3237
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
3238
      count = countObj->valueint;
3239
    } else if (countObj && countObj->type != cJSON_Number) {
3240 3241
      errorPrint("%s() LN%d, failed to read json, column count not found\n",
          __func__, __LINE__);
3242 3243 3244 3245 3246
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3247
    // column info
3248 3249
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
3250 3251 3252 3253
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d: failed to read json, column type not found\n",
          __func__, __LINE__);
3254 3255
      goto PARSE_OVER;
    }
3256 3257
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3258

3259 3260
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3261
      columnCase.dataLen = dataLen->valueint;
3262
    } else if (dataLen && dataLen->type != cJSON_Number) {
3263 3264
      debugPrint("%s() LN%d: failed to read json, column len not found\n",
          __func__, __LINE__);
3265 3266 3267 3268
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
3269

3270
    for (int n = 0; n < count; ++n) {
3271 3272 3273
      tstrncpy(superTbls->columns[index].dataType,
              columnCase.dataType, MAX_TB_NAME_SIZE);
      superTbls->columns[index].dataLen = columnCase.dataLen;
3274 3275
      index++;
    }
3276
  }
3277

3278 3279
  if ((index + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n",
3280 3281 3282 3283
            __func__, __LINE__, MAX_COLUMN_COUNT);
    goto PARSE_OVER;
  }

3284
  superTbls->columnCount = index;
3285

3286 3287
  count = 1;
  index = 0;
3288
  // tags
3289 3290
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
3291 3292
    errorPrint("%s() LN%d, failed to read json, tags not found\n",
        __func__, __LINE__);
3293 3294
    goto PARSE_OVER;
  }
3295

3296 3297
  int tagSize = cJSON_GetArraySize(tags);
  if (tagSize > MAX_TAG_COUNT) {
3298 3299
    errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n",
        __func__, __LINE__, MAX_TAG_COUNT);
3300 3301
    goto PARSE_OVER;
  }
3302 3303

  //superTbls->tagCount = tagSize;
3304 3305 3306
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
3307

3308 3309 3310
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
3311
      count = countObj->valueint;
3312
    } else if (countObj && countObj->type != cJSON_Number) {
3313
      printf("ERROR: failed to read json, column count not found\n");
3314 3315 3316 3317 3318
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3319
    // column info
3320 3321
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
3322 3323 3324 3325
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d, failed to read json, tag type not found\n",
          __func__, __LINE__);
3326 3327
      goto PARSE_OVER;
    }
3328
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3329

3330 3331
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3332
      columnCase.dataLen = dataLen->valueint;
3333
    } else if (dataLen && dataLen->type != cJSON_Number) {
3334 3335
      errorPrint("%s() LN%d, failed to read json, column len not found\n",
          __func__, __LINE__);
3336 3337 3338
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
3339 3340
    }

3341
    for (int n = 0; n < count; ++n) {
3342 3343
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType,
          MAX_TB_NAME_SIZE);
3344
      superTbls->tags[index].dataLen = columnCase.dataLen;
3345 3346
      index++;
    }
3347
  }
3348 3349

  if (index > MAX_TAG_COUNT) {
3350
    errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n",
3351 3352 3353 3354
        __func__, __LINE__, MAX_TAG_COUNT);
    goto PARSE_OVER;
  }

3355 3356
  superTbls->tagCount = index;

3357 3358 3359
  if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n",
        __func__, __LINE__, MAX_COLUMN_COUNT);
3360 3361
    goto PARSE_OVER;
  }
3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372
  ret = true;

PARSE_OVER:
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3373
    tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3374 3375 3376 3377
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3378
    tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE);
3379
  } else if (!host) {
3380
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3381
  } else {
3382
    printf("ERROR: failed to read json, host not found\n");
3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_Dbs.port = port->valueint;
  } else if (!port) {
    g_Dbs.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3395
    tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE);
3396
  } else if (!user) {
3397
    tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE);
3398 3399 3400 3401
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3402
    tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE);
3403
  } else if (!password) {
3404
    tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE);
3405 3406 3407 3408
  }

  cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
  if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
3409
    tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
3410
  } else if (!resultfile) {
3411
    tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
3412 3413 3414 3415 3416 3417 3418 3419
  }

  cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
  if (threads && threads->type == cJSON_Number) {
    g_Dbs.threadCount = threads->valueint;
  } else if (!threads) {
    g_Dbs.threadCount = 1;
  } else {
3420
    printf("ERROR: failed to read json, threads not found\n");
3421
    goto PARSE_OVER;
3422 3423
  }

3424 3425 3426 3427
  cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
  if (threads2 && threads2->type == cJSON_Number) {
    g_Dbs.threadCountByCreateTbl = threads2->valueint;
  } else if (!threads2) {
3428
    g_Dbs.threadCountByCreateTbl = 1;
3429
  } else {
3430 3431
    errorPrint("%s() LN%d, failed to read json, threads2 not found\n",
            __func__, __LINE__);
3432
    goto PARSE_OVER;
3433
  }
3434

3435 3436
  cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
  if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
3437 3438 3439 3440 3441
    if (gInsertInterval->valueint <0) {
      errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n",
            __func__, __LINE__);
      goto PARSE_OVER;
    }
3442 3443 3444 3445
    g_args.insert_interval = gInsertInterval->valueint;
  } else if (!gInsertInterval) {
    g_args.insert_interval = 0;
  } else {
3446 3447
    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
        __func__, __LINE__);
3448 3449
    goto PARSE_OVER;
  }
3450

3451 3452
  cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows");
  if (interlaceRows && interlaceRows->type == cJSON_Number) {
3453 3454 3455 3456 3457 3458
    if (interlaceRows->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;

    }
3459 3460
    g_args.interlace_rows = interlaceRows->valueint;
  } else if (!interlaceRows) {
3461
    g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3462
  } else {
3463 3464
    errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
        __func__, __LINE__);
3465
    goto PARSE_OVER;
3466
  }
3467

3468 3469
  cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
  if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
3470 3471 3472 3473 3474
    if (maxSqlLen->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
    }
3475 3476
    g_args.max_sql_len = maxSqlLen->valueint;
  } else if (!maxSqlLen) {
3477
    g_args.max_sql_len = (1024*1024);
3478
  } else {
3479 3480
    errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
        __func__, __LINE__);
3481 3482 3483 3484 3485
    goto PARSE_OVER;
  }

  cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
  if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
3486
    if (numRecPerReq->valueint <= 0) {
3487 3488 3489
      errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
3490
    } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) {
3491 3492 3493 3494 3495
      printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n",
              numRecPerReq->valueint, MAX_RECORDS_PER_REQ);
      printf("        number of records per request value will be set to %d\n\n",
              MAX_RECORDS_PER_REQ);
      prompt();
3496
      numRecPerReq->valueint = MAX_RECORDS_PER_REQ;
3497
    }
3498 3499
    g_args.num_of_RPR = numRecPerReq->valueint;
  } else if (!numRecPerReq) {
3500
    g_args.num_of_RPR = MAX_RECORDS_PER_REQ;
3501
  } else {
3502 3503
    errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n",
        __func__, __LINE__);
3504 3505 3506
    goto PARSE_OVER;
  }

3507
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3508
  if (answerPrompt
3509 3510
          && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3511 3512 3513 3514 3515 3516 3517 3518
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
    } else {
      g_args.answer_yes = false;
    }
  } else if (!answerPrompt) {
3519
    g_args.answer_yes = true;   // default is no, mean answer_yes.
3520
  } else {
3521
    errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n");
3522
    goto PARSE_OVER;
3523
  }
3524

3525 3526 3527 3528 3529 3530 3531 3532 3533 3534
  // rows per table need be less than insert batch
  if (g_args.interlace_rows > g_args.num_of_RPR) {
      printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
              g_args.interlace_rows, g_args.num_of_RPR);
      printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
              g_args.num_of_RPR);
      prompt();
      g_args.interlace_rows = g_args.num_of_RPR;
  }

3535 3536
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (!dbs || dbs->type != cJSON_Array) {
3537
    printf("ERROR: failed to read json, databases not found\n");
3538 3539 3540 3541 3542
    goto PARSE_OVER;
  }

  int dbSize = cJSON_GetArraySize(dbs);
  if (dbSize > MAX_DB_COUNT) {
3543
    errorPrint(
3544 3545
            "ERROR: failed to read json, databases size overflow, max database is %d\n",
            MAX_DB_COUNT);
3546 3547 3548 3549 3550 3551 3552 3553
    goto PARSE_OVER;
  }

  g_Dbs.dbCount = dbSize;
  for (int i = 0; i < dbSize; ++i) {
    cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
    if (dbinfos == NULL) continue;

3554
    // dbinfo
3555 3556
    cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
    if (!dbinfo || dbinfo->type != cJSON_Object) {
3557
      printf("ERROR: failed to read json, dbinfo not found\n");
3558 3559
      goto PARSE_OVER;
    }
3560

3561 3562
    cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
    if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
3563
      printf("ERROR: failed to read json, db name not found\n");
3564 3565
      goto PARSE_OVER;
    }
3566
    tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE);
3567 3568 3569

    cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
    if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
3570 3571
      if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) {
        g_Dbs.db[i].drop = true;
3572
      } else {
3573
        g_Dbs.db[i].drop = false;
3574
      }
3575
    } else if (!drop) {
3576
      g_Dbs.db[i].drop = g_args.drop_database;
3577
    } else {
3578 3579
      errorPrint("%s() LN%d, failed to read json, drop input mistake\n",
              __func__, __LINE__);
3580 3581 3582 3583
      goto PARSE_OVER;
    }

    cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
3584 3585 3586 3587
    if (precision && precision->type == cJSON_String
            && precision->valuestring != NULL) {
      tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
              MAX_DB_NAME_SIZE);
3588
    } else if (!precision) {
3589
      //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
3590 3591
      memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE);
    } else {
3592
      printf("ERROR: failed to read json, precision not found\n");
3593 3594 3595 3596 3597 3598 3599 3600 3601
      goto PARSE_OVER;
    }

    cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
    if (update && update->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.update = update->valueint;
    } else if (!update) {
      g_Dbs.db[i].dbCfg.update = -1;
    } else {
3602
      printf("ERROR: failed to read json, update not found\n");
3603 3604 3605 3606 3607 3608 3609 3610 3611
      goto PARSE_OVER;
    }

    cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
    if (replica && replica->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.replica = replica->valueint;
    } else if (!replica) {
      g_Dbs.db[i].dbCfg.replica = -1;
    } else {
3612
      printf("ERROR: failed to read json, replica not found\n");
3613 3614 3615 3616 3617 3618 3619 3620 3621
      goto PARSE_OVER;
    }

    cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
    if (keep && keep->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.keep = keep->valueint;
    } else if (!keep) {
      g_Dbs.db[i].dbCfg.keep = -1;
    } else {
3622
     printf("ERROR: failed to read json, keep not found\n");
3623 3624
     goto PARSE_OVER;
    }
3625

3626 3627 3628 3629 3630 3631
    cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
    if (days && days->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.days = days->valueint;
    } else if (!days) {
      g_Dbs.db[i].dbCfg.days = -1;
    } else {
3632
     printf("ERROR: failed to read json, days not found\n");
3633 3634
     goto PARSE_OVER;
    }
3635

3636 3637 3638 3639 3640 3641
    cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
    if (cache && cache->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cache = cache->valueint;
    } else if (!cache) {
      g_Dbs.db[i].dbCfg.cache = -1;
    } else {
3642
     printf("ERROR: failed to read json, cache not found\n");
3643 3644
     goto PARSE_OVER;
    }
3645

3646 3647 3648 3649 3650 3651
    cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
    if (blocks && blocks->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
    } else if (!blocks) {
      g_Dbs.db[i].dbCfg.blocks = -1;
    } else {
3652
     printf("ERROR: failed to read json, block not found\n");
3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669
     goto PARSE_OVER;
    }

    //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
    //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
    //} else if (!maxtablesPerVnode) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
    //} else {
    // printf("failed to read json, maxtablesPerVnode not found");
    // goto PARSE_OVER;
    //}

    cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
    if (minRows && minRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
    } else if (!minRows) {
3670
      g_Dbs.db[i].dbCfg.minRows = 0;    // 0 means default
3671
    } else {
3672
     printf("ERROR: failed to read json, minRows not found\n");
3673 3674 3675 3676 3677 3678 3679
     goto PARSE_OVER;
    }

    cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
    if (maxRows && maxRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
    } else if (!maxRows) {
3680
      g_Dbs.db[i].dbCfg.maxRows = 0;    // 0 means default
3681
    } else {
3682
     printf("ERROR: failed to read json, maxRows not found\n");
3683 3684 3685 3686 3687 3688 3689 3690 3691
     goto PARSE_OVER;
    }

    cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
    if (comp && comp->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.comp = comp->valueint;
    } else if (!comp) {
      g_Dbs.db[i].dbCfg.comp = -1;
    } else {
3692
     printf("ERROR: failed to read json, comp not found\n");
3693 3694 3695 3696 3697 3698 3699 3700 3701
     goto PARSE_OVER;
    }

    cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
    if (walLevel && walLevel->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
    } else if (!walLevel) {
      g_Dbs.db[i].dbCfg.walLevel = -1;
    } else {
3702
     printf("ERROR: failed to read json, walLevel not found\n");
3703 3704 3705
     goto PARSE_OVER;
    }

3706 3707 3708 3709 3710 3711
    cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
    if (cacheLast && cacheLast->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
    } else if (!cacheLast) {
      g_Dbs.db[i].dbCfg.cacheLast = -1;
    } else {
3712
     printf("ERROR: failed to read json, cacheLast not found\n");
3713 3714 3715
     goto PARSE_OVER;
    }

3716 3717 3718 3719
    cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
    if (quorum && quorum->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
    } else if (!quorum) {
3720
      g_Dbs.db[i].dbCfg.quorum = 1;
3721
    } else {
3722
     printf("failed to read json, quorum input mistake");
3723 3724 3725 3726 3727 3728 3729 3730 3731
     goto PARSE_OVER;
    }

    cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
    if (fsync && fsync->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
    } else if (!fsync) {
      g_Dbs.db[i].dbCfg.fsync = -1;
    } else {
3732 3733 3734
      errorPrint("%s() LN%d, failed to read json, fsync input mistake\n",
              __func__, __LINE__);
      goto PARSE_OVER;
3735
    }
3736

3737
    // super_talbes
3738 3739
    cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
    if (!stables || stables->type != cJSON_Array) {
3740 3741
      errorPrint("%s() LN%d, failed to read json, super_tables not found\n",
              __func__, __LINE__);
3742
      goto PARSE_OVER;
3743 3744
    }

3745 3746
    int stbSize = cJSON_GetArraySize(stables);
    if (stbSize > MAX_SUPER_TABLE_COUNT) {
3747
      errorPrint(
3748 3749
              "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n",
              __func__, __LINE__, MAX_SUPER_TABLE_COUNT);
3750 3751 3752 3753 3754 3755 3756
      goto PARSE_OVER;
    }

    g_Dbs.db[i].superTblCount = stbSize;
    for (int j = 0; j < stbSize; ++j) {
      cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
      if (stbInfo == NULL) continue;
3757 3758

      // dbinfo
3759 3760
      cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
      if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) {
3761 3762
        errorPrint("%s() LN%d, failed to read json, stb name not found\n",
                __func__, __LINE__);
3763 3764
        goto PARSE_OVER;
      }
3765
      tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE);
3766

3767 3768
      cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
      if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
3769
        printf("ERROR: failed to read json, childtable_prefix not found\n");
3770 3771
        goto PARSE_OVER;
      }
3772
      tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE);
3773 3774

      cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null
3775 3776 3777
      if (autoCreateTbl
              && autoCreateTbl->type == cJSON_String
              && autoCreateTbl->valuestring != NULL) {
3778 3779 3780 3781 3782 3783 3784 3785 3786 3787
        if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
        } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        } else {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        }
      } else if (!autoCreateTbl) {
        g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      } else {
3788
        printf("ERROR: failed to read json, auto_create_table not found\n");
3789 3790
        goto PARSE_OVER;
      }
3791

3792 3793 3794 3795
      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
      } else if (!batchCreateTbl) {
3796
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
3797
      } else {
3798
        printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
3799
        goto PARSE_OVER;
3800
      }
3801 3802

      cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
3803 3804 3805
      if (childTblExists
              && childTblExists->type == cJSON_String
              && childTblExists->valuestring != NULL) {
3806 3807
        if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3))
            && (g_Dbs.db[i].drop == false)) {
3808
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
3809 3810
        } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2)
              || (g_Dbs.db[i].drop == true))) {
3811 3812 3813 3814 3815 3816 3817
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        } else {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        }
      } else if (!childTblExists) {
        g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
      } else {
3818 3819
        errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n",
                __func__, __LINE__);
3820 3821
        goto PARSE_OVER;
      }
3822

3823 3824
      cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
      if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
3825
        errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n",
3826
                __func__, __LINE__);
3827 3828 3829 3830 3831
        goto PARSE_OVER;
      }
      g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

      cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
3832 3833 3834 3835
      if (dataSource && dataSource->type == cJSON_String
              && dataSource->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
                dataSource->valuestring, MAX_DB_NAME_SIZE);
3836
      } else if (!dataSource) {
3837
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE);
3838
      } else {
3839 3840
        errorPrint("%s() LN%d, failed to read json, data_source not found\n",
            __func__, __LINE__);
3841 3842 3843
        goto PARSE_OVER;
      }

3844
      cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest
3845 3846 3847 3848
      if (insertMode && insertMode->type == cJSON_String
              && insertMode->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode,
                insertMode->valuestring, MAX_DB_NAME_SIZE);
3849
      } else if (!insertMode) {
3850
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE);
3851
      } else {
3852
        printf("ERROR: failed to read json, insert_mode not found\n");
3853 3854 3855
        goto PARSE_OVER;
      }

3856
      cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
3857 3858
      if ((childTbl_limit) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3859 3860 3861 3862 3863 3864
        if (childTbl_limit->type != cJSON_Number) {
            printf("ERROR: failed to read json, childtable_limit\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
      } else {
3865
        g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid.
3866 3867 3868
      }

      cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
3869 3870
      if ((childTbl_offset) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3871 3872 3873 3874 3875 3876 3877 3878 3879
        if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) {
            printf("ERROR: failed to read json, childtable_offset\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblOffset = 0;
      }

3880 3881
      cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
      if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
3882 3883
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                ts->valuestring, MAX_DB_NAME_SIZE);
3884
      } else if (!ts) {
3885 3886
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                "now", MAX_DB_NAME_SIZE);
3887
      } else {
3888
        printf("ERROR: failed to read json, start_timestamp not found\n");
3889 3890
        goto PARSE_OVER;
      }
3891

3892 3893 3894 3895
      cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
      if (timestampStep && timestampStep->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
      } else if (!timestampStep) {
3896
        g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
3897
      } else {
3898
        printf("ERROR: failed to read json, timestamp_step not found\n");
3899 3900 3901 3902
        goto PARSE_OVER;
      }

      cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
3903 3904
      if (sampleFormat && sampleFormat->type
              == cJSON_String && sampleFormat->valuestring != NULL) {
3905 3906
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
                sampleFormat->valuestring, MAX_DB_NAME_SIZE);
3907
      } else if (!sampleFormat) {
3908
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE);
3909
      } else {
3910
        printf("ERROR: failed to read json, sample_format not found\n");
3911
        goto PARSE_OVER;
3912
      }
3913

3914
      cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
3915 3916
      if (sampleFile && sampleFile->type == cJSON_String
          && sampleFile->valuestring != NULL) {
3917 3918
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
                sampleFile->valuestring, MAX_FILE_NAME_LEN);
3919 3920 3921
      } else if (!sampleFile) {
        memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
      } else {
3922
        printf("ERROR: failed to read json, sample_file not found\n");
3923
        goto PARSE_OVER;
3924
      }
3925

3926 3927
      cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
      if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) {
3928 3929
        tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
                tagsFile->valuestring, MAX_FILE_NAME_LEN);
3930 3931 3932 3933 3934 3935 3936 3937 3938
        if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
          g_Dbs.db[i].superTbls[j].tagSource = 0;
        } else {
          g_Dbs.db[i].superTbls[j].tagSource = 1;
        }
      } else if (!tagsFile) {
        memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
        g_Dbs.db[i].superTbls[j].tagSource = 0;
      } else {
3939
        printf("ERROR: failed to read json, tags_file not found\n");
3940 3941
        goto PARSE_OVER;
      }
3942

3943 3944 3945 3946 3947
      cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
      if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
        int32_t len = maxSqlLen->valueint;
        if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
          len = TSDB_MAX_ALLOWED_SQL_LEN;
3948 3949
        } else if (len < 5) {
          len = 5;
3950
        }
3951 3952
        g_Dbs.db[i].superTbls[j].maxSqlLen = len;
      } else if (!maxSqlLen) {
3953
        g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len;
3954
      } else {
3955 3956
        errorPrint("%s() LN%d, failed to read json, maxSqlLen input mistake\n",
            __func__, __LINE__);
3957
        goto PARSE_OVER;
3958
      }
3959
/*
3960 3961 3962 3963 3964
      cJSON *multiThreadWriteOneTbl =
          cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
      if (multiThreadWriteOneTbl
              && multiThreadWriteOneTbl->type == cJSON_String
              && multiThreadWriteOneTbl->valuestring != NULL) {
3965 3966 3967 3968
        if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
        } else {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
3969
        }
3970 3971 3972
      } else if (!multiThreadWriteOneTbl) {
        g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
      } else {
3973
        printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
3974 3975
        goto PARSE_OVER;
      }
3976
*/
3977 3978
      cJSON* interlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows");
      if (interlaceRows && interlaceRows->type == cJSON_Number) {
3979 3980 3981 3982 3983
        if (interlaceRows->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n",
            __func__, __LINE__);
          goto PARSE_OVER;
        }
3984
        g_Dbs.db[i].superTbls[j].interlaceRows = interlaceRows->valueint;
3985 3986
        // rows per table need be less than insert batch
        if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) {
3987
          printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
3988
                  i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR);
3989
          printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
3990
                  g_args.num_of_RPR);
3991
          prompt();
3992 3993
          g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR;
        }
3994 3995
      } else if (!interlaceRows) {
        g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3996
      } else {
3997 3998 3999
        errorPrint(
                "%s() LN%d, failed to read json, interlace rows input mistake\n",
                __func__, __LINE__);
4000
        goto PARSE_OVER;
4001
      }
4002 4003 4004

      cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
      if (disorderRatio && disorderRatio->type == cJSON_Number) {
4005 4006 4007 4008 4009 4010
        if (disorderRatio->valueint > 50)
          disorderRatio->valueint = 50;

        if (disorderRatio->valueint < 0)
          disorderRatio->valueint = 0;

4011 4012 4013 4014
        g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
      } else if (!disorderRatio) {
        g_Dbs.db[i].superTbls[j].disorderRatio = 0;
      } else {
4015
        printf("ERROR: failed to read json, disorderRatio not found\n");
4016
        goto PARSE_OVER;
4017
      }
4018 4019 4020 4021 4022 4023 4024

      cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
      if (disorderRange && disorderRange->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
      } else if (!disorderRange) {
        g_Dbs.db[i].superTbls[j].disorderRange = 1000;
      } else {
4025
        printf("ERROR: failed to read json, disorderRange not found\n");
4026 4027
        goto PARSE_OVER;
      }
4028

4029 4030
      cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
      if (insertRows && insertRows->type == cJSON_Number) {
4031 4032 4033 4034 4035
        if (insertRows->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4036 4037 4038 4039
        g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
      } else if (!insertRows) {
        g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
      } else {
4040 4041
        errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
4042 4043 4044 4045 4046 4047
        goto PARSE_OVER;
      }

      cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
      if (insertInterval && insertInterval->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
4048 4049 4050 4051 4052
        if (insertInterval->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4053
      } else if (!insertInterval) {
4054
        verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n",
4055 4056 4057
                __func__, __LINE__, g_args.insert_interval);
        g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
      } else {
4058 4059
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
4060 4061 4062
        goto PARSE_OVER;
      }

4063 4064
      int retVal = getColumnAndTagTypeFromInsertJsonFile(
              stbInfo, &g_Dbs.db[i].superTbls[j]);
4065 4066
      if (false == retVal) {
        goto PARSE_OVER;
4067 4068
      }
    }
4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081
  }

  ret = true;

PARSE_OVER:
  return ret;
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
4082
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
4083 4084 4085 4086
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
4087
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE);
4088
  } else if (!host) {
4089
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
4090
  } else {
4091
    printf("ERROR: failed to read json, host not found\n");
4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
4104
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE);
4105
  } else if (!user) {
4106
    tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ;
4107 4108 4109 4110
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
4111
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE);
4112
  } else if (!password) {
4113
    tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);;
4114
  }
H
hzcheng 已提交
4115

4116
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
4117 4118
  if (answerPrompt && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
4119 4120 4121 4122
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
4123
    } else {
4124
      g_args.answer_yes = false;
H
hzcheng 已提交
4125
    }
4126 4127 4128
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
4129
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
4130
    goto PARSE_OVER;
4131
  }
4132

4133 4134
  cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
  if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
4135
    if (gQueryTimes->valueint <= 0) {
4136 4137
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4138 4139
      goto PARSE_OVER;
    }
4140 4141 4142 4143
    g_args.query_times = gQueryTimes->valueint;
  } else if (!gQueryTimes) {
    g_args.query_times = 1;
  } else {
4144 4145
    errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4146 4147 4148
    goto PARSE_OVER;
  }

4149 4150
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
4151
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE);
4152
  } else if (!dbs) {
4153
    printf("ERROR: failed to read json, databases not found\n");
4154
    goto PARSE_OVER;
H
hzcheng 已提交
4155
  }
4156 4157 4158

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
4159
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
4160
  } else if (!queryMode) {
4161
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
4162
  } else {
4163
    printf("ERROR: failed to read json, query_mode not found\n");
4164
    goto PARSE_OVER;
H
hzcheng 已提交
4165
  }
4166

4167
  // specified_table_query
4168 4169
  cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!specifiedQuery) {
4170
    g_queryInfo.specifiedQueryInfo.concurrent = 1;
4171 4172
    g_queryInfo.specifiedQueryInfo.sqlCount = 0;
  } else if (specifiedQuery->type != cJSON_Object) {
4173
    printf("ERROR: failed to read json, super_table_query not found\n");
4174
    goto PARSE_OVER;
4175
  } else {
4176 4177 4178 4179 4180
    cJSON* queryInterval = cJSON_GetObjectItem(specifiedQuery, "query_interval");
    if (queryInterval && queryInterval->type == cJSON_Number) {
      g_queryInfo.specifiedQueryInfo.queryInterval = queryInterval->valueint;
    } else if (!queryInterval) {
      g_queryInfo.specifiedQueryInfo.queryInterval = 0;
4181 4182
    }

4183 4184
    cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery,
        "query_times");
4185
    if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
4186
      if (specifiedQueryTimes->valueint <= 0) {
4187 4188
        errorPrint(
                "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4189
          __func__, __LINE__, specifiedQueryTimes->valueint);
4190 4191 4192
        goto PARSE_OVER;

      }
4193 4194 4195 4196
      g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint;
    } else if (!specifiedQueryTimes) {
      g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times;
    } else {
4197 4198
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
          __func__, __LINE__);
4199
      goto PARSE_OVER;
4200
    }
4201

4202
    cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent");
4203
    if (concurrent && concurrent->type == cJSON_Number) {
4204
      if (concurrent->valueint <= 0) {
4205
        errorPrint(
4206
                "%s() LN%d, query sqlCount %"PRIu64" or concurrent %d is not correct.\n",
4207 4208
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount,
4209 4210 4211
              g_queryInfo.specifiedQueryInfo.concurrent);
        goto PARSE_OVER;
      }
4212
      g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint;
4213
    } else if (!concurrent) {
4214
      g_queryInfo.specifiedQueryInfo.concurrent = 1;
4215
    }
4216

4217 4218 4219 4220 4221 4222 4223
    cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode");
    if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String
        && specifiedAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE;
4224
      } else {
4225
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4226
            __func__, __LINE__);
4227 4228 4229
        goto PARSE_OVER;
      }
    } else {
4230
      g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
4231
    }
4232

4233
    cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval");
4234
    if (interval && interval->type == cJSON_Number) {
4235
      g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint;
4236
    } else if (!interval) {
4237 4238
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4239
      g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000;
4240
    }
4241

4242
    cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart");
4243
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
4244
      if (0 == strcmp("yes", restart->valuestring)) {
4245
        g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4246
      } else if (0 == strcmp("no", restart->valuestring)) {
4247
        g_queryInfo.specifiedQueryInfo.subscribeRestart = false;
4248
      } else {
4249
        printf("ERROR: failed to read json, subscribe restart error\n");
4250 4251 4252
        goto PARSE_OVER;
      }
    } else {
4253
      g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4254
    }
4255

4256
    cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress");
4257 4258 4259
    if (keepProgress
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
4260
      if (0 == strcmp("yes", keepProgress->valuestring)) {
4261
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1;
4262
      } else if (0 == strcmp("no", keepProgress->valuestring)) {
4263
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4264
      } else {
4265
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4266 4267 4268
        goto PARSE_OVER;
      }
    } else {
4269
      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4270 4271
    }

4272
    // sqls
4273 4274
    cJSON* specifiedSqls = cJSON_GetObjectItem(specifiedQuery, "sqls");
    if (!specifiedSqls) {
4275
      g_queryInfo.specifiedQueryInfo.sqlCount = 0;
4276
    } else if (specifiedSqls->type != cJSON_Array) {
4277 4278
      errorPrint("%s() LN%d, failed to read json, super sqls not found\n",
          __func__, __LINE__);
4279
      goto PARSE_OVER;
4280
    } else {
4281 4282 4283 4284 4285 4286 4287 4288
      int superSqlSize = cJSON_GetArraySize(specifiedSqls);
      if (superSqlSize * g_queryInfo.specifiedQueryInfo.concurrent
              > MAX_QUERY_SQL_COUNT) {
        errorPrint("%s() LN%d, failed to read json, query sql(%d) * concurrent(%d) overflow, max is %d\n",
           __func__, __LINE__,
           superSqlSize,
           g_queryInfo.specifiedQueryInfo.concurrent,
           MAX_QUERY_SQL_COUNT);
4289 4290
        goto PARSE_OVER;
      }
4291

4292
      g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize;
4293
      for (int j = 0; j < superSqlSize; ++j) {
4294
        cJSON* sql = cJSON_GetArrayItem(specifiedSqls, j);
4295
        if (sql == NULL) continue;
4296

4297 4298
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
4299
          printf("ERROR: failed to read json, sql not found\n");
4300 4301
          goto PARSE_OVER;
        }
4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315
        tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j],
                sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);

        cJSON* resubAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume");
        if (resubAfterConsume
                && resubAfterConsume->type == cJSON_Number) {
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j]
                = resubAfterConsume->valueint;
        } else if (!resubAfterConsume) {
            //printf("failed to read json, subscribe interval no found\n");
            //goto PARSE_OVER;
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = 1;
        }
H
hzcheng 已提交
4316

4317
        cJSON *result = cJSON_GetObjectItem(sql, "result");
4318 4319 4320 4321
        if ((NULL != result) && (result->type == cJSON_String)
                && (result->valuestring != NULL)) {
          tstrncpy(g_queryInfo.specifiedQueryInfo.result[j],
                  result->valuestring, MAX_FILE_NAME_LEN);
4322
        } else if (NULL == result) {
4323 4324
          memset(g_queryInfo.specifiedQueryInfo.result[j],
                  0, MAX_FILE_NAME_LEN);
4325
        } else {
4326
          printf("ERROR: failed to read json, super query result file not found\n");
4327
          goto PARSE_OVER;
4328
        }
4329 4330 4331
      }
    }
  }
4332

4333
  // super_table_query
4334 4335
  cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!superQuery) {
4336
    g_queryInfo.superQueryInfo.threadCnt = 1;
4337 4338
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
4339
    printf("ERROR: failed to read json, sub_table_query not found\n");
4340 4341
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
4342
  } else {
4343
    cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval");
4344
    if (subrate && subrate->type == cJSON_Number) {
4345
      g_queryInfo.superQueryInfo.queryInterval = subrate->valueint;
4346
    } else if (!subrate) {
4347
      g_queryInfo.superQueryInfo.queryInterval = 0;
4348 4349 4350 4351
    }

    cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
    if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
4352 4353 4354
      if (superQueryTimes->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
          __func__, __LINE__, superQueryTimes->valueint);
4355 4356
        goto PARSE_OVER;
      }
4357 4358 4359 4360
      g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
    } else if (!superQueryTimes) {
      g_queryInfo.superQueryInfo.queryTimes = g_args.query_times;
    } else {
4361 4362
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
          __func__, __LINE__);
4363
      goto PARSE_OVER;
4364
    }
4365

4366
    cJSON* threads = cJSON_GetObjectItem(superQuery, "threads");
4367
    if (threads && threads->type == cJSON_Number) {
4368 4369 4370 4371 4372 4373
      if (threads->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, threads input mistake\n",
          __func__, __LINE__);
        goto PARSE_OVER;

      }
4374
      g_queryInfo.superQueryInfo.threadCnt = threads->valueint;
4375
    } else if (!threads) {
4376
      g_queryInfo.superQueryInfo.threadCnt = 1;
4377
    }
4378

4379
    //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count");
4380
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
4381
    //  g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint;
4382
    //} else if (!subTblCnt) {
4383
    //  g_queryInfo.superQueryInfo.childTblCount = 0;
4384
    //}
4385

4386
    cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname");
4387 4388 4389 4390
    if (stblname && stblname->type == cJSON_String
        && stblname->valuestring != NULL) {
      tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring,
          MAX_TB_NAME_SIZE);
4391
    } else {
4392 4393
      errorPrint("%s() LN%d, failed to read json, super table name input error\n",
          __func__, __LINE__);
4394 4395
      goto PARSE_OVER;
    }
4396

4397 4398 4399 4400 4401 4402 4403
    cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode");
    if (superAsyncMode && superAsyncMode->type == cJSON_String
        && superAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE;
4404
      } else {
4405
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4406
            __func__, __LINE__);
4407 4408 4409
        goto PARSE_OVER;
      }
    } else {
4410
      g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
4411
    }
4412

4413 4414 4415 4416 4417 4418 4419 4420 4421
    cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval");
    if (superInterval && superInterval->type == cJSON_Number) {
      if (superInterval->valueint < 0) {
        errorPrint("%s() LN%d, failed to read json, interval input mistake\n",
            __func__, __LINE__);
        goto PARSE_OVER;
      }
      g_queryInfo.superQueryInfo.subscribeInterval = superInterval->valueint;
    } else if (!superInterval) {
4422 4423
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4424
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
4425
    }
4426

4427
    cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart");
4428 4429
    if (subrestart && subrestart->type == cJSON_String
        && subrestart->valuestring != NULL) {
4430
      if (0 == strcmp("yes", subrestart->valuestring)) {
4431
        g_queryInfo.superQueryInfo.subscribeRestart = true;
4432
      } else if (0 == strcmp("no", subrestart->valuestring)) {
4433
        g_queryInfo.superQueryInfo.subscribeRestart = false;
4434
      } else {
4435
        printf("ERROR: failed to read json, subscribe restart error\n");
4436 4437 4438
        goto PARSE_OVER;
      }
    } else {
4439
      g_queryInfo.superQueryInfo.subscribeRestart = true;
4440
    }
4441

4442 4443 4444 4445 4446
    cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
    if (superkeepProgress &&
            superkeepProgress->type == cJSON_String
            && superkeepProgress->valuestring != NULL) {
      if (0 == strcmp("yes", superkeepProgress->valuestring)) {
4447
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
4448
      } else if (0 == strcmp("no", superkeepProgress->valuestring)) {
4449
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4450
      } else {
4451
        printf("ERROR: failed to read json, subscribe super table keepProgress error\n");
4452 4453 4454
        goto PARSE_OVER;
      }
    } else {
4455
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4456
    }
4457

4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469
    cJSON* superResubAfterConsume =
            cJSON_GetObjectItem(superQuery, "resubAfterConsume");
    if (superResubAfterConsume
            && superResubAfterConsume->type == cJSON_Number) {
        g_queryInfo.superQueryInfo.resubAfterConsume =
            superResubAfterConsume->valueint;
    } else if (!superResubAfterConsume) {
        //printf("failed to read json, subscribe interval no found\n");
        ////goto PARSE_OVER;
        g_queryInfo.superQueryInfo.resubAfterConsume = 1;
    }

4470 4471 4472
    // supert table sqls
    cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls");
    if (!superSqls) {
4473
      g_queryInfo.superQueryInfo.sqlCount = 0;
4474
    } else if (superSqls->type != cJSON_Array) {
4475 4476
      errorPrint("%s() LN%d: failed to read json, super sqls not found\n",
          __func__, __LINE__);
4477
      goto PARSE_OVER;
4478
    } else {
4479
      int superSqlSize = cJSON_GetArraySize(superSqls);
4480
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4481 4482
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4483 4484
        goto PARSE_OVER;
      }
4485

4486
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
4487
      for (int j = 0; j < superSqlSize; ++j) {
4488
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
4489
        if (sql == NULL) continue;
4490

4491
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
4492 4493 4494 4495
        if (!sqlStr || sqlStr->type != cJSON_String
            || sqlStr->valuestring == NULL) {
          errorPrint("%s() LN%d, failed to read json, sql not found\n",
              __func__, __LINE__);
4496 4497
          goto PARSE_OVER;
        }
4498 4499
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring,
            MAX_QUERY_SQL_LENGTH);
4500 4501

        cJSON *result = cJSON_GetObjectItem(sql, "result");
4502 4503 4504 4505
        if (result != NULL && result->type == cJSON_String
            && result->valuestring != NULL){
          tstrncpy(g_queryInfo.superQueryInfo.result[j],
              result->valuestring, MAX_FILE_NAME_LEN);
4506
        } else if (NULL == result) {
4507
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4508
        }  else {
4509 4510
          errorPrint("%s() LN%d, failed to read json, sub query result file not found\n",
              __func__, __LINE__);
4511
          goto PARSE_OVER;
4512
        }
4513 4514
      }
    }
H
hzcheng 已提交
4515 4516
  }

4517
  ret = true;
H
hzcheng 已提交
4518

4519 4520 4521
PARSE_OVER:
  return ret;
}
H
hzcheng 已提交
4522

4523
static bool getInfoFromJsonFile(char* file) {
4524
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
4525

4526 4527 4528 4529 4530
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
4531

4532
  bool  ret = false;
4533
  int   maxLen = 6400000;
4534 4535 4536 4537 4538 4539 4540
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
4541
  }
H
Hui Li 已提交
4542

4543 4544 4545
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
4546
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
4547 4548
    goto PARSE_OVER;
  }
H
Hui Li 已提交
4549

4550 4551 4552
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
4553
      g_args.test_mode = INSERT_TEST;
4554
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
4555
      g_args.test_mode = QUERY_TEST;
4556
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
4557
      g_args.test_mode = SUBSCRIBE_TEST;
4558
    } else {
4559
      printf("ERROR: failed to read json, filetype not support\n");
4560 4561 4562
      goto PARSE_OVER;
    }
  } else if (!filetype) {
4563
    g_args.test_mode = INSERT_TEST;
4564
  } else {
4565
    printf("ERROR: failed to read json, filetype not found\n");
4566 4567
    goto PARSE_OVER;
  }
H
hzcheng 已提交
4568

4569
  if (INSERT_TEST == g_args.test_mode) {
4570
    ret = getMetaFromInsertJsonFile(root);
4571 4572
  } else if ((QUERY_TEST == g_args.test_mode)
          || (SUBSCRIBE_TEST == g_args.test_mode)) {
4573 4574
    ret = getMetaFromQueryJsonFile(root);
  } else {
4575 4576
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n",
            __func__, __LINE__);
4577
    goto PARSE_OVER;
4578
  }
4579 4580 4581 4582

PARSE_OVER:
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
4583
  fclose(fp);
4584 4585
  return ret;
}
H
hzcheng 已提交
4586

4587
static void prepareSampleData() {
4588
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4589 4590 4591
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
        (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]);
S
Shuaiqiang Chang 已提交
4592
      }
4593 4594 4595 4596
    }
  }
}

4597
static void postFreeResource() {
4598
  tmfclose(g_fpOfInsertResult);
4599
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4600
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
4601 4602 4603
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
4604
      }
4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4617
    }
4618 4619
  }
}
S
Shuaiqiang Chang 已提交
4620

4621 4622 4623
static int getRowDataFromSample(
        char* dataBuf, int64_t maxLen, int64_t timestamp,
      SSuperTable* superTblInfo, int64_t* sampleUsePos) {
4624
  if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
4625
/*    int ret = readSampleFromCsvFileToMem(superTblInfo);
4626
    if (0 != ret) {
4627
      tmfree(superTblInfo->sampleDataBuf);
4628
      superTblInfo->sampleDataBuf = NULL;
4629 4630
      return -1;
    }
4631
*/
4632 4633
    *sampleUsePos = 0;
  }
S
Shuaiqiang Chang 已提交
4634

4635
  int    dataLen = 0;
4636

4637 4638 4639 4640
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "(%" PRId64 ", ", timestamp);
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
4641
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
S
Shuaiqiang Chang 已提交
4642

4643
  (*sampleUsePos)++;
4644

4645 4646
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4647

4648 4649
static int64_t generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stbInfo) {
  int64_t   dataLen = 0;
4650
  char  *pstr = recBuf;
4651
  int64_t maxLen = MAX_DATA_SIZE;
4652

4653
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "(%" PRId64 ",", timestamp);
4654

4655
  for (int i = 0; i < stbInfo->columnCount; i++) {
4656 4657
    if ((0 == strncasecmp(stbInfo->columns[i].dataType, "BINARY", strlen("BINARY")))
            || (0 == strncasecmp(stbInfo->columns[i].dataType, "NCHAR", strlen("NCHAR")))) {
4658
      if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
4659
        errorPrint( "binary or nchar length overflow, max size:%u\n",
4660
                (uint32_t)TSDB_MAX_BINARY_LEN);
4661
        return -1;
4662
      }
4663

4664 4665
      char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
      if (NULL == buf) {
4666
        errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
4667
        return -1;
4668 4669
      }
      rand_string(buf, stbInfo->columns[i].dataLen);
4670
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf);
4671
      tmfree(buf);
4672
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4673
                "INT", 3)) {
4674
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4675
              "%d,", rand_int());
4676
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4677
                "BIGINT", 6)) {
4678
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4679
              "%"PRId64",", rand_bigint());
4680
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4681
                "FLOAT", 5)) {
4682
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4683
              "%f,", rand_float());
4684
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4685
                "DOUBLE", 6)) {
4686
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4687
              "%f,", rand_double());
4688
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4689
                "SMALLINT", 8)) {
4690
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4691
          "%d,", rand_smallint());
4692
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4693
          "TINYINT", strlen("TINYINT"))) {
4694
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4695
          "%d,", rand_tinyint());
4696
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4697
          "BOOL", strlen("BOOL"))) {
4698
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4699
          "%d,", rand_bool());
4700
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4701
          "TIMESTAMP", strlen("TIMESTAMP"))) {
4702
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4703
          "%"PRId64",", rand_bigint());
4704
    }  else {
4705
      errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType);
4706
      return -1;
4707
    }
S
Shuaiqiang Chang 已提交
4708
  }
4709

4710
  dataLen -= 1;
4711
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")");
4712

4713 4714 4715
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

  return strlen(recBuf);
4716
}
S
Shuaiqiang Chang 已提交
4717

4718
static int64_t generateData(char *recBuf, char **data_type,
4719
        int num_of_cols, int64_t timestamp, int lenOfBinary) {
4720 4721
  memset(recBuf, 0, MAX_DATA_SIZE);
  char *pstr = recBuf;
4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735
  pstr += sprintf(pstr, "(%" PRId64, timestamp);
  int c = 0;

  for (; c < MAX_NUM_DATATYPE; c++) {
    if (data_type[c] == NULL) {
      break;
    }
  }

  if (0 == c) {
    perror("data type error!");
    exit(-1);
  }

4736
  for (int i = 0; i < c; i++) {
4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747
    if (strcasecmp(data_type[i % c], "TINYINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_tinyint() );
    } else if (strcasecmp(data_type[i % c], "SMALLINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_smallint());
    } else if (strcasecmp(data_type[i % c], "INT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_int());
    } else if (strcasecmp(data_type[i % c], "BIGINT") == 0) {
      pstr += sprintf(pstr, ",%" PRId64, rand_bigint());
    } else if (strcasecmp(data_type[i % c], "FLOAT") == 0) {
      pstr += sprintf(pstr, ",%10.4f", rand_float());
    } else if (strcasecmp(data_type[i % c], "DOUBLE") == 0) {
4748
      double t = rand_double();
4749 4750
      pstr += sprintf(pstr, ",%20.8f", t);
    } else if (strcasecmp(data_type[i % c], "BOOL") == 0) {
4751
      bool b = rand_bool() & 1;
4752 4753
      pstr += sprintf(pstr, ",%s", b ? "true" : "false");
    } else if (strcasecmp(data_type[i % c], "BINARY") == 0) {
4754 4755
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4756
      pstr += sprintf(pstr, ",\"%s\"", s);
4757
      free(s);
4758
    } else if (strcasecmp(data_type[i % c], "NCHAR") == 0) {
4759 4760
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4761
      pstr += sprintf(pstr, ",\"%s\"", s);
4762 4763 4764
      free(s);
    }

4765
    if (strlen(recBuf) > MAX_DATA_SIZE) {
4766 4767 4768 4769 4770 4771 4772
      perror("column length too long, abort");
      exit(-1);
    }
  }

  pstr += sprintf(pstr, ")");

4773 4774
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

4775
  return (int32_t)strlen(recBuf);
4776 4777
}

4778
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
4779 4780
  char* sampleDataBuf = NULL;

4781
  sampleDataBuf = calloc(
4782
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
4783
  if (sampleDataBuf == NULL) {
4784
      errorPrint("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n",
4785
              __func__, __LINE__,
4786
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE,
4787
              strerror(errno));
4788
      return -1;
4789
  }
4790

4791 4792 4793 4794
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
4795 4796
      errorPrint("%s() LN%d, read sample from csv file failed.\n",
          __func__, __LINE__);
4797
      tmfree(sampleDataBuf);
4798
      superTblInfo->sampleDataBuf = NULL;
4799
      return -1;
H
Hui Li 已提交
4800
  }
4801

4802 4803 4804
  return 0;
}

4805
static int64_t execInsert(threadInfo *pThreadInfo, char *buffer, uint64_t k)
4806 4807
{
  int affectedRows;
4808
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4809

4810 4811
  verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
            __func__, __LINE__, buffer);
4812
  if (superTblInfo) {
4813
    if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) {
4814
      affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false);
4815
    } else if (0 == strncasecmp(superTblInfo->insertMode, "rest", strlen("rest"))) {
4816 4817
      if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port,
                  buffer, NULL /* not set result file */)) {
4818
        affectedRows = -1;
4819 4820
        printf("========restful return fail, threadID[%d]\n",
            pThreadInfo->threadID);
4821 4822 4823
      } else {
        affectedRows = k;
      }
4824 4825 4826 4827
    } else {
      errorPrint("%s() LN%d: unknown insert mode: %s\n",
        __func__, __LINE__, superTblInfo->insertMode);
      affectedRows = 0;
4828
    }
4829
  } else {
4830
    affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false);
4831 4832 4833 4834 4835
  }

  return affectedRows;
}

4836 4837
static void getTableName(char *pTblName,
        threadInfo* pThreadInfo, uint64_t tableSeq)
4838 4839
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4840 4841
  if ((superTblInfo)
          && (AUTO_CREATE_SUBTBL != superTblInfo->autoCreateTable)) {
4842
    if (superTblInfo->childTblLimit > 0) {
4843
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
4844 4845
            superTblInfo->childTblName +
            (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
4846
    } else {
4847

4848
        verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n",
4849 4850
                pThreadInfo->threadID, __func__, __LINE__,
                pThreadInfo->start_table_from,
4851 4852 4853 4854 4855
                pThreadInfo->ntables, tableSeq);
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
    }
  } else {
4856
    snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"",
4857
        g_args.tb_prefix, tableSeq);
4858 4859 4860
  }
}

4861
static int64_t generateDataTail(
4862
        SSuperTable* superTblInfo,
4863
        uint64_t batch, char* buffer, int64_t remainderBufLen, int64_t insertRows,
4864
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) {
4865 4866
  uint64_t len = 0;
  uint32_t ncols_per_record = 1; // count first col ts
4867

4868 4869
  char *pstr = buffer;

4870
  if (superTblInfo == NULL) {
4871
    uint32_t datatypeSeq = 0;
4872 4873 4874 4875
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
4876 4877
  }

4878
  verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch);
4879

4880
  uint64_t k = 0;
4881
  for (k = 0; k < batch;) {
4882
    char data[MAX_DATA_SIZE];
4883 4884
    memset(data, 0, MAX_DATA_SIZE);

4885
    int64_t retLen = 0;
4886

4887 4888
    if (superTblInfo) {
      if (0 == strncasecmp(superTblInfo->dataSource,
4889 4890
                    "sample", strlen("sample"))) {
          retLen = getRowDataFromSample(
4891 4892
                    data,
                    remainderBufLen,
4893
                    startTime + superTblInfo->timeStampStep * k,
4894
                    superTblInfo,
4895
                    pSamplePos);
4896
      } else if (0 == strncasecmp(superTblInfo->dataSource,
4897
                   "rand", strlen("rand"))) {
4898

4899
        int64_t randTail = superTblInfo->timeStampStep * k;
4900 4901 4902 4903
        if (superTblInfo->disorderRatio > 0) {
          int rand_num = taosRandom() % 100;
          if(rand_num < superTblInfo->disorderRatio) {
            randTail = (randTail + (taosRandom() % superTblInfo->disorderRange + 1)) * (-1);
4904
            debugPrint("rand data generated, back %"PRId64"\n", randTail);
4905
          }
4906 4907
        }

4908
        int64_t d = startTime
4909 4910
                + randTail;
        retLen = generateRowData(
4911
                      data,
4912
                      d,
4913
                      superTblInfo);
4914
      }
4915

4916 4917 4918
      if (retLen > remainderBufLen) {
        break;
      }
4919

4920
      pstr += snprintf(pstr , retLen + 1, "%s", data);
4921 4922 4923
      k++;
      len += retLen;
      remainderBufLen -= retLen;
4924
    } else {
4925 4926
      char **data_type = g_args.datatype;
      int lenOfBinary = g_args.len_of_binary;
4927

4928 4929 4930 4931 4932 4933
      int64_t randTail = DEFAULT_TIMESTAMP_STEP * k;

      if (g_args.disorderRatio != 0) {
        int rand_num = taosRandom() % 100;
        if (rand_num < g_args.disorderRatio) {
          randTail = (randTail + (taosRandom() % g_args.disorderRange + 1)) * (-1);
4934

4935 4936
          debugPrint("rand data generated, back %"PRId64"\n", randTail);
        }
4937
      } else {
4938 4939 4940 4941
        randTail = DEFAULT_TIMESTAMP_STEP * k;
      }

      retLen = generateData(data, data_type,
4942
                  ncols_per_record,
4943
                  startTime + randTail,
4944
                  lenOfBinary);
4945

4946 4947 4948
      if (len > remainderBufLen)
        break;

4949
      pstr += sprintf(pstr, "%s", data);
4950 4951 4952
      k++;
      len += retLen;
      remainderBufLen -= retLen;
4953 4954
    }

4955
    verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n",
4956
            __func__, __LINE__, len, k, buffer);
4957 4958

    startFrom ++;
4959

4960
    if (startFrom >= insertRows) {
4961
      break;
4962
    }
4963
  }
4964

4965 4966 4967
  *dataLen = len;
  return k;
}
4968

4969
static int generateSQLHead(char *tableName, int32_t tableSeq,
4970 4971
        threadInfo* pThreadInfo, SSuperTable* superTblInfo,
        char *buffer, int remainderBufLen)
4972 4973
{
  int len;
4974 4975 4976 4977

#define HEAD_BUFF_LEN    1024*24  // 16*1024 + (192+32)*2 + insert into ..
  char headBuf[HEAD_BUFF_LEN];

4978 4979 4980 4981
  if (superTblInfo) {
    if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
4982
            tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq);
4983 4984 4985 4986 4987 4988
      } else {
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
      }
      if (NULL == tagsValBuf) {
4989 4990
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n",
            __func__, __LINE__);
4991 4992 4993
        return -1;
      }

4994 4995 4996
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
4997
                  "%s.%s using %s.%s tags %s values",
4998 4999 5000 5001 5002 5003 5004
                  pThreadInfo->db_name,
                  tableName,
                  pThreadInfo->db_name,
                  superTblInfo->sTblName,
                  tagsValBuf);
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
5005 5006 5007
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
5008
                  "%s.%s values",
5009
                  pThreadInfo->db_name,
5010
                  tableName);
5011
    } else {
5012 5013 5014
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
5015
                  "%s.%s values",
5016 5017 5018 5019
                  pThreadInfo->db_name,
                  tableName);
    }
  } else {
5020 5021 5022
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
5023
                  "%s.%s values",
5024 5025 5026 5027
                  pThreadInfo->db_name,
                  tableName);
  }

5028 5029 5030 5031 5032
  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

5033 5034 5035
  return len;
}

5036
static int64_t generateInterlaceDataBuffer(
5037 5038
        char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes,
        uint64_t tableSeq,
5039
        threadInfo *pThreadInfo, char *buffer,
5040
        int64_t insertRows,
5041
        int64_t startTime,
5042
        uint64_t *pRemainderBufLen)
5043
{
5044
  assert(buffer);
5045 5046 5047 5048 5049 5050 5051 5052 5053 5054
  char *pstr = buffer;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo,
            superTblInfo, pstr, *pRemainderBufLen);

  if (headLen <= 0) {
    return 0;
  }
  // generate data buffer
5055
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n",
5056 5057 5058 5059 5060
            pThreadInfo->threadID, __func__, __LINE__, i, buffer);

  pstr += headLen;
  *pRemainderBufLen -= headLen;

5061
  int64_t dataLen = 0;
5062

5063
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%"PRIu64" batchPerTbl = %"PRIu64"\n",
5064 5065 5066 5067 5068 5069 5070 5071 5072 5073
            pThreadInfo->threadID, __func__, __LINE__,
            i, batchPerTblTimes, batchPerTbl);

  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
      startTime = taosGetTimestamp(pThreadInfo->time_precision);
    }
  } else {
      startTime = 1500000000000;
  }
5074

5075
  int64_t k = generateDataTail(
5076
    superTblInfo,
5077 5078 5079 5080
    batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
    startTime,
    &(pThreadInfo->samplePos), &dataLen);

5081
  if (k == batchPerTbl) {
5082 5083 5084
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
  } else {
5085
    debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n",
5086
            __func__, __LINE__, k, batchPerTbl);
5087 5088
    pstr -= headLen;
    pstr[0] = '\0';
5089
    k = 0;
5090 5091 5092 5093 5094
  }

  return k;
}

5095
static int64_t generateProgressiveDataBuffer(
5096
        char *tableName,
5097
        int64_t tableSeq,
5098 5099
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
5100
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos,
5101
        int64_t *pRemainderBufLen)
5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  int ncols_per_record = 1; // count first col ts

  if (superTblInfo == NULL) {
    int datatypeSeq = 0;
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
  }

  assert(buffer != NULL);
5116
  char *pstr = buffer;
5117

5118
  int64_t k = 0;
5119

5120
  memset(buffer, 0, *pRemainderBufLen);
5121

5122
  int64_t headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, superTblInfo,
5123
          buffer, *pRemainderBufLen);
5124 5125 5126 5127

  if (headLen <= 0) {
    return 0;
  }
5128
  pstr += headLen;
5129
  *pRemainderBufLen -= headLen;
5130

5131
  int64_t dataLen;
5132
  k = generateDataTail(superTblInfo,
5133
          g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom,
5134
          startTime,
5135
          pSamplePos, &dataLen);
5136

5137 5138
  return k;
}
5139

5140 5141 5142 5143 5144 5145 5146 5147 5148
static void printStatPerThread(threadInfo *pThreadInfo)
{
  fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n",
          pThreadInfo->threadID,
          pThreadInfo->totalInsertRows,
          pThreadInfo->totalAffectedRows,
          (double)(pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay/1000.0)));
}

5149
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
5150 5151
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
         pThreadInfo->threadID, __func__, __LINE__);
5152

5153
  int64_t insertRows;
5154
  uint64_t interlaceRows;
5155

5156
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5157

5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170
  if (superTblInfo) {
    insertRows = superTblInfo->insertRows;

    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    insertRows = g_args.num_of_DPT;
    interlaceRows = g_args.interlace_rows;
  }
5171

5172
  if (interlaceRows > insertRows)
5173
    interlaceRows = insertRows;
5174

5175 5176
  if (interlaceRows > g_args.num_of_RPR)
    interlaceRows = g_args.num_of_RPR;
5177

5178 5179 5180 5181 5182 5183 5184 5185 5186 5187
  int insertMode;

  if (interlaceRows > 0) {
    insertMode = INTERLACE_INSERT_MODE;
  } else {
    insertMode = PROGRESSIVE_INSERT_MODE;
  }

  // TODO: prompt tbl count multple interlace rows and batch
  //
5188

5189
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
5190
  char* buffer = calloc(maxSqlLen, 1);
5191
  if (NULL == buffer) {
5192
    errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5193
              __func__, __LINE__, maxSqlLen, strerror(errno));
5194 5195 5196
    return NULL;
  }

5197
  char tableName[TSDB_TABLE_NAME_LEN];
5198 5199 5200 5201

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

5202
  int64_t nTimeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
5203

5204
  uint64_t insert_interval =
5205
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5206 5207
  uint64_t st = 0;
  uint64_t et = UINT64_MAX;
5208

5209 5210 5211
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5212

5213
  uint64_t tableSeq = pThreadInfo->start_table_from;
5214

5215
  debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n",
5216
          pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from,
5217 5218 5219 5220
          pThreadInfo->ntables, insertRows);

  int64_t startTime = pThreadInfo->start_time;

5221 5222
  uint64_t batchPerTbl = interlaceRows;
  uint64_t batchPerTblTimes;
5223

5224
  if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) {
5225
    batchPerTblTimes =
5226
        g_args.num_of_RPR / interlaceRows;
5227 5228 5229 5230
  } else {
    batchPerTblTimes = 1;
  }

5231
  uint64_t generatedRecPerTbl = 0;
5232
  bool flagSleep = true;
5233
  uint64_t sleepTimeTotal = 0;
5234

5235 5236 5237
  char *strInsertInto = "insert into ";
  int nInsertBufLen = strlen(strInsertInto);

5238
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
5239
    if ((flagSleep) && (insert_interval)) {
5240
        st = taosGetTimestampMs();
5241
        flagSleep = false;
5242 5243
    }
    // generate data
5244
    memset(buffer, 0, maxSqlLen);
5245
    uint64_t remainderBufLen = maxSqlLen;
5246

5247
    char *pstr = buffer;
5248 5249 5250 5251 5252

    int len = snprintf(pstr, nInsertBufLen + 1, "%s", strInsertInto);
    pstr += len;
    remainderBufLen -= len;

5253
    uint64_t recOfBatch = 0;
5254

5255
    for (uint64_t i = 0; i < batchPerTblTimes; i ++) {
5256
      getTableName(tableName, pThreadInfo, tableSeq);
5257 5258 5259
      if (0 == strlen(tableName)) {
        errorPrint("[%d] %s() LN%d, getTableName return null\n",
            pThreadInfo->threadID, __func__, __LINE__);
5260
        free(buffer);
5261 5262
        return NULL;
      }
5263

5264
      uint64_t oldRemainderLen = remainderBufLen;
5265
      int64_t generated = generateInterlaceDataBuffer(
5266 5267 5268 5269
        tableName, batchPerTbl, i, batchPerTblTimes,
        tableSeq,
        pThreadInfo, pstr,
        insertRows,
5270
        startTime,
5271
        &remainderBufLen);
5272

5273 5274
      debugPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5275
      if (generated < 0) {
5276
        errorPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
5277
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5278
        goto free_of_interlace;
5279 5280
      } else if (generated == 0) {
        break;
5281 5282
      }

5283
      tableSeq ++;
5284
      recOfBatch += batchPerTbl;
5285
      pstr += (oldRemainderLen - remainderBufLen);
5286
//      startTime += batchPerTbl * superTblInfo->timeStampStep;
5287
      pThreadInfo->totalInsertRows += batchPerTbl;
5288
      verbosePrint("[%d] %s() LN%d batchPerTbl=%"PRId64" recOfBatch=%"PRId64"\n",
5289 5290
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
5291

5292 5293 5294 5295
      if (insertMode == INTERLACE_INSERT_MODE) {
          if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
            // turn to first table
            tableSeq = pThreadInfo->start_table_from;
5296
            generatedRecPerTbl += batchPerTbl;
5297 5298

            startTime = pThreadInfo->start_time
5299
              + generatedRecPerTbl * nTimeStampStep;
5300

5301 5302 5303 5304
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;

5305 5306 5307 5308
            int remainRows = insertRows - generatedRecPerTbl;
            if ((remainRows > 0) && (batchPerTbl > remainRows))
              batchPerTbl = remainRows;

5309 5310
            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
5311
          }
5312 5313
      }

5314
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n",
5315 5316 5317 5318
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
5319 5320 5321
        break;
    }

5322
    verbosePrint("[%d] %s() LN%d recOfBatch=%"PRIu64" totalInsertRows=%"PRIu64"\n",
5323 5324 5325 5326 5327
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
           pThreadInfo->threadID, __func__, __LINE__, buffer);

5328
    startTs = taosGetTimestampMs();
5329

5330 5331 5332 5333 5334 5335 5336
    if (recOfBatch == 0) {
      errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n",
              pThreadInfo->threadID, __func__, __LINE__,
              recOfBatch);
      errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n");
      goto free_of_interlace;
    }
5337
    int64_t affectedRows = execInsert(pThreadInfo, buffer, recOfBatch);
5338

5339
    endTs = taosGetTimestampMs();
5340 5341
    uint64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n",
5342
            __func__, __LINE__, delay);
5343 5344 5345
    verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5346 5347 5348 5349 5350 5351

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

5352 5353
    if (recOfBatch != affectedRows) {
        errorPrint("[%d] %s() LN%d execInsert insert %"PRIu64", affected rows: %"PRId64"\n%s\n",
5354
                pThreadInfo->threadID, __func__, __LINE__,
5355
                recOfBatch, affectedRows, buffer);
5356
        goto free_of_interlace;
5357
    }
5358

5359
    pThreadInfo->totalAffectedRows += affectedRows;
5360

5361 5362
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
5363
      printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n",
5364 5365 5366
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5367 5368
      lastPrintTime = currentPrintTime;
    }
5369

5370
    if ((insert_interval) && flagSleep) {
5371
      et = taosGetTimestampMs();
5372

5373 5374
      if (insert_interval > (et - st) ) {
        int sleepTime = insert_interval - (et -st);
5375 5376
        performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleepTime);
5377 5378 5379
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
5380
    }
5381 5382
  }

5383
free_of_interlace:
5384
  tmfree(buffer);
5385
  printStatPerThread(pThreadInfo);
5386 5387 5388
  return NULL;
}

5389 5390 5391 5392 5393 5394 5395 5396
// sync insertion
/*
   1 thread: 100 tables * 2000  rows/s
   1 thread: 10  tables * 20000 rows/s
   6 thread: 300 tables * 2000  rows/s

   2 taosinsertdata , 1 thread:  10  tables * 20000 rows/s
*/
5397
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
5398
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
5399

5400
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5401
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
5402

5403
  char* buffer = calloc(maxSqlLen, 1);
5404
  if (NULL == buffer) {
5405
    errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5406
              maxSqlLen,
5407 5408 5409
              strerror(errno));
    return NULL;
  }
5410

5411 5412 5413
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5414

5415
  int64_t timeStampStep =
5416
      superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
5417
/*  int insert_interval =
5418
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5419 5420
  uint64_t st = 0;
  uint64_t et = 0xffffffff;
5421
  */
5422

5423 5424
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
5425

5426
  pThreadInfo->samplePos = 0;
5427

5428
  for (uint64_t tableSeq =
5429
          pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to;
5430
        tableSeq ++) {
5431
    int64_t start_time = pThreadInfo->start_time;
5432

5433
    int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
5434

5435 5436
    verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows);

5437
    for (uint64_t i = 0; i < insertRows;) {
5438
        /*
5439
      if (insert_interval) {
5440
            st = taosGetTimestampMs();
5441
      }
5442
      */
5443

5444 5445
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
5446
      verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n",
5447 5448 5449
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);

5450
      int64_t remainderBufLen = maxSqlLen;
5451 5452 5453 5454 5455 5456 5457 5458
      char *pstr = buffer;
      int nInsertBufLen = strlen("insert into ");

      int len = snprintf(pstr, nInsertBufLen + 1, "%s", "insert into ");

      pstr += len;
      remainderBufLen -= len;

5459
      int64_t generated = generateProgressiveDataBuffer(
5460
              tableName, tableSeq, pThreadInfo, pstr, insertRows,
5461
            i, start_time,
5462 5463
            &(pThreadInfo->samplePos),
            &remainderBufLen);
5464 5465 5466
      if (generated > 0)
        i += generated;
      else
5467
        goto free_of_progressive;
5468

5469
      start_time +=  generated * timeStampStep;
5470
      pThreadInfo->totalInsertRows += generated;
5471

5472
      startTs = taosGetTimestampMs();
5473

5474
      int64_t affectedRows = execInsert(pThreadInfo, buffer, generated);
5475

5476
      endTs = taosGetTimestampMs();
5477
      uint64_t delay = endTs - startTs;
5478 5479
      performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n",
              __func__, __LINE__, delay);
5480 5481 5482
      verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5483

5484 5485 5486 5487
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
5488

5489 5490 5491 5492 5493
      if (affectedRows < 0) {
        errorPrint("%s() LN%d, affected rows: %"PRId64"\n",
                __func__, __LINE__, affectedRows);
        goto free_of_progressive;
      }
5494 5495 5496

      pThreadInfo->totalAffectedRows += affectedRows;

5497 5498
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
5499
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
5500 5501 5502
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5503 5504 5505
        lastPrintTime = currentPrintTime;
      }

5506
      if (i >= insertRows)
5507
        break;
5508
/*
5509
      if (insert_interval) {
5510
        et = taosGetTimestampMs();
5511

5512 5513
        if (insert_interval > ((et - st)) ) {
            int sleep_time = insert_interval - (et -st);
5514 5515
            performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleep_time);
5516 5517
            taosMsleep(sleep_time); // ms
        }
5518
      }
5519
      */
5520 5521
    }   // num_of_DPT

5522 5523
    if (g_args.verbose_print) {
      if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo &&
5524 5525
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
5526
          verbosePrint("%s() LN%d samplePos=%"PRId64"\n",
5527
                  __func__, __LINE__, pThreadInfo->samplePos);
5528
      }
5529
    }
5530
  } // tableSeq
5531

5532
free_of_progressive:
5533
  tmfree(buffer);
5534
  printStatPerThread(pThreadInfo);
5535
  return NULL;
H
Hui Li 已提交
5536 5537
}

5538 5539
static void* syncWrite(void *sarg) {

5540 5541
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5542

5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554
  int interlaceRows;

  if (superTblInfo) {
    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    interlaceRows = g_args.interlace_rows;
  }
5555

5556
  if (interlaceRows > 0) {
5557
    // interlace mode
5558
    return syncWriteInterlace(pThreadInfo);
5559 5560
  } else {
    // progressive mode
5561
    return syncWriteProgressive(pThreadInfo);
5562
  }
5563

5564 5565
}

5566
static void callBack(void *param, TAOS_RES *res, int code) {
5567 5568
  threadInfo* pThreadInfo = (threadInfo*)param;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5569

5570 5571
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5572
  if (insert_interval) {
5573 5574 5575
    pThreadInfo->et = taosGetTimestampMs();
    if ((pThreadInfo->et - pThreadInfo->st) < insert_interval) {
      taosMsleep(insert_interval - (pThreadInfo->et - pThreadInfo->st)); // ms
5576
    }
H
Hui Li 已提交
5577
  }
5578

5579
  char *buffer = calloc(1, pThreadInfo->superTblInfo->maxSqlLen);
5580
  char data[MAX_DATA_SIZE];
5581
  char *pstr = buffer;
5582 5583
  pstr += sprintf(pstr, "insert into %s.%s%"PRId64" values",
          pThreadInfo->db_name, pThreadInfo->tb_prefix,
5584 5585 5586 5587 5588 5589 5590 5591
          pThreadInfo->start_table_from);
//  if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
  if (pThreadInfo->counter >= g_args.num_of_RPR) {
    pThreadInfo->start_table_from++;
    pThreadInfo->counter = 0;
  }
  if (pThreadInfo->start_table_from > pThreadInfo->end_table_to) {
    tsem_post(&pThreadInfo->lock_sem);
5592 5593 5594 5595
    free(buffer);
    taos_free_result(res);
    return;
  }
5596

5597
  for (int i = 0; i < g_args.num_of_RPR; i++) {
5598
    int rand_num = taosRandom() % 100;
5599 5600
    if (0 != pThreadInfo->superTblInfo->disorderRatio
            && rand_num < pThreadInfo->superTblInfo->disorderRatio) {
5601 5602
      int64_t d = pThreadInfo->lastTs
          - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1);
5603
      generateRowData(data, d, pThreadInfo->superTblInfo);
5604
    } else {
5605
      generateRowData(data, pThreadInfo->lastTs += 1000, pThreadInfo->superTblInfo);
H
Hui Li 已提交
5606
    }
5607
    pstr += sprintf(pstr, "%s", data);
5608
    pThreadInfo->counter++;
H
Hui Li 已提交
5609

5610
    if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
5611
      break;
H
Hui Li 已提交
5612 5613
    }
  }
5614

5615
  if (insert_interval) {
5616
    pThreadInfo->st = taosGetTimestampMs();
5617
  }
5618
  taos_query_a(pThreadInfo->taos, buffer, callBack, pThreadInfo);
5619
  free(buffer);
H
Hui Li 已提交
5620

5621
  taos_free_result(res);
H
Hui Li 已提交
5622 5623
}

5624
static void *asyncWrite(void *sarg) {
5625 5626
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5627

5628 5629 5630
  pThreadInfo->st = 0;
  pThreadInfo->et = 0;
  pThreadInfo->lastTs = pThreadInfo->start_time;
5631

5632
  int insert_interval =
5633
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5634
  if (insert_interval) {
5635
    pThreadInfo->st = taosGetTimestampMs();
H
Hui Li 已提交
5636
  }
5637
  taos_query_a(pThreadInfo->taos, "show databases", callBack, pThreadInfo);
5638

5639
  tsem_wait(&(pThreadInfo->lock_sem));
H
Hui Li 已提交
5640 5641 5642 5643

  return NULL;
}

5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669
static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in *serv_addr)
{
  uint16_t rest_port = port + TSDB_PORT_HTTP;
  struct hostent *server = gethostbyname(host);
  if ((server == NULL) || (server->h_addr == NULL)) {
    errorPrint("%s", "ERROR, no such host");
    return -1;
  }

  debugPrint("h_name: %s\nh_addr=%p\nh_addretype: %s\nh_length: %d\n",
            server->h_name,
            server->h_addr,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

  memset(serv_addr, 0, sizeof(struct sockaddr_in));
  serv_addr->sin_family = AF_INET;
  serv_addr->sin_port = htons(rest_port);
#ifdef WINDOWS
  serv_addr->sin_addr.s_addr = inet_addr(host);
#else
  memcpy(&(serv_addr->sin_addr.s_addr), server->h_addr, server->h_length);
#endif
  return 0;
}

5670 5671
static void startMultiThreadInsertData(int threads, char* db_name,
        char* precision,SSuperTable* superTblInfo) {
5672

5673 5674
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  assert(pids != NULL);
H
Hui Li 已提交
5675

5676 5677
  threadInfo *infos = malloc(threads * sizeof(threadInfo));
  assert(infos != NULL);
5678

5679 5680
  memset(pids, 0, threads * sizeof(pthread_t));
  memset(infos, 0, threads * sizeof(threadInfo));
5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697

  //TAOS* taos;
  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
  //  if (NULL == taos) {
  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
  //    exit(-1);
  //  }
  //}

  int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
  if (0 != precision[0]) {
    if (0 == strncasecmp(precision, "ms", 2)) {
      timePrec = TSDB_TIME_PRECISION_MILLI;
    }  else if (0 == strncasecmp(precision, "us", 2)) {
      timePrec = TSDB_TIME_PRECISION_MICRO;
    }  else {
5698
      errorPrint("Not support precision: %s\n", precision);
5699 5700 5701 5702
      exit(-1);
    }
  }

5703
  int64_t start_time;
5704 5705 5706
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
        start_time = taosGetTimestamp(timePrec);
5707
    } else {
5708 5709 5710 5711 5712
      if (TSDB_CODE_SUCCESS != taosParseTime(
        superTblInfo->startTimestamp,
        &start_time,
        strlen(superTblInfo->startTimestamp),
        timePrec, 0)) {
5713
          ERROR_EXIT("failed to parse time!\n");
5714
      }
5715
    }
5716 5717
  } else {
     start_time = 1500000000000;
5718 5719
  }

5720
  int64_t start = taosGetTimestampMs();
5721

5722
  // read sample data from file first
5723
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5724
              "sample", strlen("sample")))) {
5725
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5726 5727
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
              __func__, __LINE__);
5728 5729 5730 5731
      exit(-1);
    }
  }

5732
  // read sample data from file first
5733
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5734 5735
              "sample", strlen("sample")))) {
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5736 5737
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
              __func__, __LINE__);
5738 5739 5740 5741 5742 5743 5744 5745
      exit(-1);
    }
  }

  TAOS* taos = taos_connect(
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
  if (NULL == taos) {
5746 5747
    errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
                __func__, __LINE__, taos_errstr(NULL));
5748 5749 5750
    exit(-1);
  }

5751
  int64_t ntables = 0;
5752
  uint64_t startFrom;
5753

5754
  if (superTblInfo) {
5755 5756
    int64_t limit;
    uint64_t offset;
5757

5758
    if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) &&
5759
            ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) {
5760
      printf("WARNING: offset and limit will not be used since the child tables not exists!\n");
5761 5762
    }

5763
    if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) {
5764 5765
      if ((superTblInfo->childTblLimit < 0)
          || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit)
5766
            > (superTblInfo->childTblCount))) {
5767 5768 5769 5770 5771 5772
        superTblInfo->childTblLimit =
            superTblInfo->childTblCount - superTblInfo->childTblOffset;
      }

      offset = superTblInfo->childTblOffset;
      limit = superTblInfo->childTblLimit;
5773
    } else {
5774 5775
      limit = superTblInfo->childTblCount;
      offset = 0;
5776 5777
    }

5778 5779 5780
    ntables = limit;
    startFrom = offset;

5781 5782 5783 5784
    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
        && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
            > superTblInfo->childTblCount)) {
      printf("WARNING: specified offset + limit > child table count!\n");
5785
      prompt();
5786 5787 5788 5789 5790
    }

    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
            && (0 == superTblInfo->childTblLimit)) {
      printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
5791
      prompt();
5792 5793
    }

5794 5795 5796
    superTblInfo->childTblName = (char*)calloc(1,
        limit * TSDB_TABLE_NAME_LEN);
    if (superTblInfo->childTblName == NULL) {
5797
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
5798 5799 5800 5801
      taos_close(taos);
      exit(-1);
    }

5802
    int64_t childTblCount;
5803 5804 5805 5806 5807 5808
    getChildNameOfSuperTableWithLimitAndOffset(
        taos,
        db_name, superTblInfo->sTblName,
        &superTblInfo->childTblName, &childTblCount,
        limit,
        offset);
5809 5810 5811
  } else {
    ntables = g_args.num_of_tables;
    startFrom = 0;
5812
  }
5813

5814 5815
  taos_close(taos);

5816
  int64_t a = ntables / threads;
5817 5818 5819 5820 5821
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

5822
  int64_t b = 0;
5823 5824 5825 5826
  if (threads != 0) {
    b = ntables % threads;
  }

5827 5828 5829 5830 5831 5832
  if ((superTblInfo)
      && (0 == strncasecmp(superTblInfo->insertMode, "rest", strlen("rest")))) {
    if (convertHostToServAddr(g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0)
      exit(-1);
  }

H
Hui Li 已提交
5833
  for (int i = 0; i < threads; i++) {
5834 5835 5836 5837 5838
    threadInfo *pThreadInfo = infos + i;
    pThreadInfo->threadID = i;
    tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE);
    pThreadInfo->time_precision = timePrec;
    pThreadInfo->superTblInfo = superTblInfo;
5839

5840 5841
    pThreadInfo->start_time = start_time;
    pThreadInfo->minDelay = UINT64_MAX;
5842

5843 5844
    if ((NULL == superTblInfo) ||
            (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) {
5845 5846
      //pThreadInfo->taos = taos;
      pThreadInfo->taos = taos_connect(
5847
              g_Dbs.host, g_Dbs.user,
5848
              g_Dbs.password, db_name, g_Dbs.port);
5849
      if (NULL == pThreadInfo->taos) {
5850 5851
        errorPrint(
                "connect to server fail from insert sub thread, reason: %s\n",
5852
                taos_errstr(NULL));
5853 5854 5855
        exit(-1);
      }
    } else {
5856
      pThreadInfo->taos = NULL;
5857 5858
    }

5859
/*    if ((NULL == superTblInfo)
5860
            || (0 == superTblInfo->multiThreadWriteOneTbl)) {
5861
            */
5862 5863 5864 5865
      pThreadInfo->start_table_from = startFrom;
      pThreadInfo->ntables = i<b?a+1:a;
      pThreadInfo->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = pThreadInfo->end_table_to + 1;
5866
/*    } else {
5867 5868 5869
      pThreadInfo->start_table_from = 0;
      pThreadInfo->ntables = superTblInfo->childTblCount;
      pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint();
5870
    }
5871
*/
5872
    tsem_init(&(pThreadInfo->lock_sem), 0, 0);
5873
    if (ASYNC_MODE == g_Dbs.asyncMode) {
5874
      pthread_create(pids + i, NULL, asyncWrite, pThreadInfo);
5875
    } else {
5876
      pthread_create(pids + i, NULL, syncWrite, pThreadInfo);
5877
    }
H
Hui Li 已提交
5878
  }
5879

H
Hui Li 已提交
5880 5881 5882 5883
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

5884 5885 5886 5887
  uint64_t totalDelay = 0;
  uint64_t maxDelay = 0;
  uint64_t minDelay = UINT64_MAX;
  uint64_t cntDelay = 1;
5888 5889
  double  avgDelay = 0;

H
Hui Li 已提交
5890
  for (int i = 0; i < threads; i++) {
5891
    threadInfo *pThreadInfo = infos + i;
5892

5893 5894
    tsem_destroy(&(pThreadInfo->lock_sem));
    taos_close(pThreadInfo->taos);
5895

5896
    debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n",
5897
            __func__, __LINE__,
5898 5899
            pThreadInfo->threadID, pThreadInfo->totalInsertRows,
            pThreadInfo->totalAffectedRows);
5900
    if (superTblInfo) {
5901 5902
        superTblInfo->totalAffectedRows += pThreadInfo->totalAffectedRows;
        superTblInfo->totalInsertRows += pThreadInfo->totalInsertRows;
5903
    } else {
5904 5905
        g_args.totalAffectedRows += pThreadInfo->totalAffectedRows;
        g_args.totalInsertRows += pThreadInfo->totalInsertRows;
5906
    }
5907

5908 5909 5910 5911
    totalDelay  += pThreadInfo->totalDelay;
    cntDelay   += pThreadInfo->cntDelay;
    if (pThreadInfo->maxDelay > maxDelay) maxDelay = pThreadInfo->maxDelay;
    if (pThreadInfo->minDelay < minDelay) minDelay = pThreadInfo->minDelay;
H
Hui Li 已提交
5912
  }
5913
  cntDelay -= 1;
H
Hui Li 已提交
5914

5915
  if (cntDelay == 0)    cntDelay = 1;
5916 5917
  avgDelay = (double)totalDelay / cntDelay;

5918 5919
  int64_t end = taosGetTimestampMs();
  int64_t t = end - start;
5920 5921

  if (superTblInfo) {
5922
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
5923
          t / 1000.0, superTblInfo->totalInsertRows,
5924 5925
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5926
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
5927 5928 5929 5930

    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
5931
          t / 1000.0, superTblInfo->totalInsertRows,
5932 5933
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5934
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
5935
    }
5936
  } else {
5937
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
5938
          t / 1000.0, g_args.totalInsertRows,
5939 5940
          g_args.totalAffectedRows,
          threads, db_name,
5941
          (double)g_args.totalInsertRows / (t / 1000.0));
5942 5943 5944
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
5945
          t * 1000.0, g_args.totalInsertRows,
5946 5947
          g_args.totalAffectedRows,
          threads, db_name,
5948
          (double)g_args.totalInsertRows / (t / 1000.0));
5949
    }
5950
  }
5951

5952
  fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
5953
          avgDelay, maxDelay, minDelay);
5954 5955
  if (g_fpOfInsertResult) {
    fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
5956
          avgDelay, maxDelay, minDelay);
5957
  }
5958

5959 5960
  //taos_close(taos);

H
Hui Li 已提交
5961
  free(pids);
5962
  free(infos);
H
Hui Li 已提交
5963 5964
}

5965
static void *readTable(void *sarg) {
5966
#if 1
5967 5968
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  TAOS *taos = pThreadInfo->taos;
H
hzcheng 已提交
5969
  char command[BUFFER_SIZE] = "\0";
5970 5971 5972
  uint64_t sTime = pThreadInfo->start_time;
  char *tb_prefix = pThreadInfo->tb_prefix;
  FILE *fp = fopen(pThreadInfo->filePath, "a");
H
Hui Li 已提交
5973
  if (NULL == fp) {
5974
    errorPrint( "fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno));
H
Hui Li 已提交
5975 5976
    return NULL;
  }
5977

5978
  int64_t num_of_DPT;
5979 5980
/*  if (pThreadInfo->superTblInfo) {
    num_of_DPT = pThreadInfo->superTblInfo->insertRows; //  nrecords_per_table;
5981
  } else {
5982
  */
5983
      num_of_DPT = g_args.num_of_DPT;
5984
//  }
5985

5986
  int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
5987
  int64_t totalData = num_of_DPT * num_of_tables;
5988
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
5989 5990 5991 5992 5993

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
5994
  printf("%"PRId64" records:\n", totalData);
H
hzcheng 已提交
5995 5996
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

5997
  for (int j = 0; j < n; j++) {
H
hzcheng 已提交
5998
    double totalT = 0;
5999
    uint64_t count = 0;
6000 6001
    for (int64_t i = 0; i < num_of_tables; i++) {
      sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64,
6002
              aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
6003

6004
      double t = taosGetTimestampMs();
S
Shuaiqiang Chang 已提交
6005 6006
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
6007

S
Shuaiqiang Chang 已提交
6008
      if (code != 0) {
6009
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6010
        taos_free_result(pSql);
H
hzcheng 已提交
6011
        taos_close(taos);
6012
        fclose(fp);
6013
        return NULL;
H
hzcheng 已提交
6014 6015
      }

6016
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6017 6018 6019
        count++;
      }

6020
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6021 6022
      totalT += t;

S
Shuaiqiang Chang 已提交
6023
      taos_free_result(pSql);
H
hzcheng 已提交
6024 6025
    }

6026
    fprintf(fp, "|%10s  |   %"PRId64"   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
6027
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
6028
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
6029
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000);
H
hzcheng 已提交
6030 6031 6032
  }
  fprintf(fp, "\n");
  fclose(fp);
6033
#endif
H
hzcheng 已提交
6034 6035 6036
  return NULL;
}

6037
static void *readMetric(void *sarg) {
6038
#if 1
6039 6040
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  TAOS *taos = pThreadInfo->taos;
H
hzcheng 已提交
6041
  char command[BUFFER_SIZE] = "\0";
6042
  FILE *fp = fopen(pThreadInfo->filePath, "a");
H
Hui Li 已提交
6043
  if (NULL == fp) {
6044
    printf("fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno));
H
Hui Li 已提交
6045 6046
    return NULL;
  }
6047

6048 6049
  int64_t num_of_DPT = pThreadInfo->superTblInfo->insertRows;
  int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
6050
  int64_t totalData = num_of_DPT * num_of_tables;
6051
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6052 6053 6054 6055 6056

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
6057 6058
  printf("%"PRId64" records:\n", totalData);
  fprintf(fp, "Querying On %"PRId64" records:\n", totalData);
H
hzcheng 已提交
6059 6060

  for (int j = 0; j < n; j++) {
6061
    char condition[COND_BUF_LEN] = "\0";
B
Bomin Zhang 已提交
6062
    char tempS[64] = "\0";
H
hzcheng 已提交
6063

6064
    int64_t m = 10 < num_of_tables ? 10 : num_of_tables;
H
hzcheng 已提交
6065

6066
    for (int64_t i = 1; i <= m; i++) {
H
hzcheng 已提交
6067
      if (i == 1) {
6068
        sprintf(tempS, "t1 = %"PRId64"", i);
H
hzcheng 已提交
6069
      } else {
6070
        sprintf(tempS, " or t1 = %"PRId64" ", i);
H
hzcheng 已提交
6071
      }
6072
      strncat(condition, tempS, COND_BUF_LEN - 1);
H
hzcheng 已提交
6073

L
liu0x54 已提交
6074
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
6075 6076 6077 6078

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

6079
      double t = taosGetTimestampMs();
H
hzcheng 已提交
6080

S
Shuaiqiang Chang 已提交
6081 6082 6083 6084
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
6085
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6086
        taos_free_result(pSql);
H
hzcheng 已提交
6087
        taos_close(taos);
6088
        fclose(fp);
6089
        return NULL;
H
hzcheng 已提交
6090 6091
      }
      int count = 0;
6092
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6093 6094
        count++;
      }
6095
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6096

6097
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
6098 6099
              num_of_tables * num_of_DPT / (t * 1000.0), t);
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0);
H
hzcheng 已提交
6100

S
Shuaiqiang Chang 已提交
6101
      taos_free_result(pSql);
H
hzcheng 已提交
6102 6103 6104 6105
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
6106
#endif
H
hzcheng 已提交
6107 6108 6109
  return NULL;
}

6110 6111 6112
static void prompt()
{
  if (!g_args.answer_yes) {
6113
    printf("         Press enter key to continue or Ctrl-C to stop\n\n");
6114 6115 6116 6117
    (void)getchar();
  }
}

6118
static int insertTestProcess() {
6119

6120 6121 6122
  setupForAnsiEscape();
  int ret = printfInsertMeta();
  resetAfterAnsiEscape();
6123

6124 6125 6126
  if (ret == -1)
    exit(EXIT_FAILURE);

6127
  debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
6128 6129
  g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
  if (NULL == g_fpOfInsertResult) {
6130
    errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
6131 6132
    return -1;
  }
6133

6134 6135
  if (g_fpOfInsertResult)
    printfInsertMetaToFile(g_fpOfInsertResult);
6136

6137
  prompt();
6138

6139 6140 6141
  init_rand_data();

  // create database and super tables
6142
  if(createDatabasesAndStables() != 0) {
6143 6144
    if (g_fpOfInsertResult)
      fclose(g_fpOfInsertResult);
6145 6146
    return -1;
  }
6147 6148

  // pretreatement
6149
  prepareSampleData();
6150

6151 6152 6153 6154
  double start;
  double end;

  // create child tables
6155
  start = taosGetTimestampMs();
6156
  createChildTables();
6157
  end = taosGetTimestampMs();
6158

6159
  if (g_totalChildTables > 0) {
6160
    fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6161
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6162 6163
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
6164
            "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6165
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6166
    }
6167
  }
6168

6169
  taosMsleep(1000);
6170
  // create sub threads for inserting data
6171
  //start = taosGetTimestampMs();
6172
  for (int i = 0; i < g_Dbs.dbCount; i++) {
6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
        for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {

          SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];

          if (superTblInfo && (superTblInfo->insertRows > 0)) {
            startMultiThreadInsertData(
              g_Dbs.threadCount,
              g_Dbs.db[i].dbName,
              g_Dbs.db[i].dbCfg.precision,
              superTblInfo);
          }
6186
        }
6187
      }
6188 6189
    } else {
        startMultiThreadInsertData(
6190 6191 6192
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
6193
          NULL);
H
Hui Li 已提交
6194
    }
6195
  }
6196
  //end = taosGetTimestampMs();
6197

6198
  //int64_t    totalInsertRows = 0;
6199
  //int64_t    totalAffectedRows = 0;
6200
  //for (int i = 0; i < g_Dbs.dbCount; i++) {
6201
  //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6202
  //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
6203 6204
  //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
  //}
6205
  //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
6206
  postFreeResource();
6207

6208 6209 6210
  return 0;
}

6211 6212
static void *specifiedTableQuery(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6213

6214
  if (pThreadInfo->taos == NULL) {
6215 6216 6217 6218 6219 6220 6221 6222
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6223
            pThreadInfo->threadID, taos_errstr(NULL));
6224 6225
      return NULL;
    } else {
6226
      pThreadInfo->taos = taos;
6227 6228 6229
    }
  }

6230 6231
  char sqlStr[MAX_DB_NAME_SIZE + 5];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6232 6233
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6234 6235 6236 6237
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
    return NULL;
  }
6238

6239 6240
  uint64_t st = 0;
  uint64_t et = 0;
6241

6242
  uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes;
6243

6244 6245 6246
  uint64_t totalQueried = 0;
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
6247

6248 6249
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq] != NULL) {
    sprintf(pThreadInfo->filePath, "%s-%d",
6250 6251 6252 6253
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }

6254
  while(queryTimes --) {
6255 6256 6257
    if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) <
            (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms
6258 6259
    }

6260 6261 6262
    st = taosGetTimestampMs();

    selectAndGetResult(pThreadInfo,
6263
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]);
6264

6265 6266 6267 6268
    et = taosGetTimestampMs();
    printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n",
              taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0);

6269 6270
    totalQueried ++;
    g_queryInfo.specifiedQueryInfo.totalQueried ++;
6271

6272 6273
    uint64_t  currentPrintTime = taosGetTimestampMs();
    uint64_t  endTs = taosGetTimestampMs();
6274
    if (currentPrintTime - lastPrintTime > 30*1000) {
6275
      debugPrint("%s() LN%d, endTs=%"PRIu64"ms, startTs=%"PRIu64"ms\n",
6276
          __func__, __LINE__, endTs, startTs);
6277
      printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.6f\n",
6278 6279
                    pThreadInfo->threadID,
                    totalQueried,
6280
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6281
      lastPrintTime = currentPrintTime;
6282
    }
H
Hui Li 已提交
6283
  }
6284 6285
  return NULL;
}
H
Hui Li 已提交
6286

6287
static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) {
6288 6289
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
6290 6291
  sprintf(subTblName, "%s.%s",
          g_queryInfo.dbName,
6292
          g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
6293 6294

  //printf("inSql: %s\n", inSql);
6295

6296 6297
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
6298
    return;
H
Hui Li 已提交
6299
  }
6300

6301
  tstrncpy(outSql, inSql, pos - inSql + 1);
6302
  //printf("1: %s\n", outSql);
6303
  strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1);
6304
  //printf("2: %s\n", outSql);
6305
  strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1);
6306
  //printf("3: %s\n", outSql);
H
Hui Li 已提交
6307 6308
}

6309
static void *superTableQuery(void *sarg) {
6310
  char sqlstr[MAX_QUERY_SQL_LENGTH];
6311
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6312

6313
  if (pThreadInfo->taos == NULL) {
6314 6315 6316 6317 6318 6319 6320 6321
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6322
            pThreadInfo->threadID, taos_errstr(NULL));
6323 6324
      return NULL;
    } else {
6325
      pThreadInfo->taos = taos;
6326 6327 6328
    }
  }

6329 6330
  uint64_t st = 0;
  uint64_t et = (int64_t)g_queryInfo.superQueryInfo.queryInterval;
6331

6332 6333 6334
  uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes;
  uint64_t totalQueried = 0;
  uint64_t  startTs = taosGetTimestampMs();
6335

6336
  uint64_t  lastPrintTime = taosGetTimestampMs();
6337
  while(queryTimes --) {
6338 6339 6340
    if (g_queryInfo.superQueryInfo.queryInterval
            && (et - st) < (int64_t)g_queryInfo.superQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.superQueryInfo.queryInterval - (et - st)); // ms
6341
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to);
6342
    }
H
Hui Li 已提交
6343

6344
    st = taosGetTimestampMs();
6345
    for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) {
6346
      for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
6347
        memset(sqlstr,0,sizeof(sqlstr));
6348
        replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i);
6349 6350
        if (g_queryInfo.superQueryInfo.result[j] != NULL) {
          sprintf(pThreadInfo->filePath, "%s-%d",
6351
                  g_queryInfo.superQueryInfo.result[j],
6352
                  pThreadInfo->threadID);
6353
        }
6354
        selectAndGetResult(pThreadInfo, sqlstr);
6355 6356 6357 6358 6359 6360 6361

        totalQueried++;
        g_queryInfo.superQueryInfo.totalQueried ++;

        int64_t  currentPrintTime = taosGetTimestampMs();
        int64_t  endTs = taosGetTimestampMs();
        if (currentPrintTime - lastPrintTime > 30*1000) {
6362
          printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.3f\n",
6363 6364
                    pThreadInfo->threadID,
                    totalQueried,
6365
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6366
          lastPrintTime = currentPrintTime;
6367
        }
6368
      }
H
Hui Li 已提交
6369
    }
6370
    et = taosGetTimestampMs();
6371
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%"PRIu64" - %"PRIu64"] once queries duration:%.4fs\n\n",
6372
            taosGetSelfPthreadId(),
6373 6374
            pThreadInfo->start_table_from,
            pThreadInfo->end_table_to,
6375
            (double)(et - st)/1000.0);
6376
  }
6377

6378 6379 6380
  return NULL;
}

6381
static int queryTestProcess() {
6382 6383 6384 6385

  setupForAnsiEscape();
  printfQueryMeta();
  resetAfterAnsiEscape();
6386 6387 6388 6389 6390 6391

  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
6392
          g_queryInfo.port);
6393
  if (taos == NULL) {
6394 6395
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6396 6397 6398
    exit(-1);
  }

6399
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6400 6401
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6402 6403 6404
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6405
  }
6406

6407
  prompt();
6408

6409 6410 6411
  if (g_args.debug_print || g_args.verbose_print) {
    printfQuerySystemInfo(taos);
  }
6412

6413 6414 6415 6416 6417 6418
  if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
    if (convertHostToServAddr(
        g_queryInfo.host, g_queryInfo.port, &g_queryInfo.serv_addr) != 0)
      exit(-1);
  }

6419 6420 6421
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
6422
  int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent;
6423
  uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount;
6424

6425
  uint64_t startTs = taosGetTimestampMs();
6426

6427 6428 6429 6430 6431 6432
  if ((nSqlCount > 0) && (nConcurrent > 0)) {

    pids  = malloc(nConcurrent * nSqlCount * sizeof(pthread_t));
    infos = malloc(nConcurrent * nSqlCount * sizeof(threadInfo));

    if ((NULL == pids) || (NULL == infos)) {
6433
      taos_close(taos);
6434
      ERROR_EXIT("memory allocation failed for create threads\n");
6435
    }
6436

6437 6438 6439
    for (uint64_t i = 0; i < nSqlCount; i++) {
        for (int j = 0; j < nConcurrent; j++) {
            uint64_t seq = i * nConcurrent + j;
6440 6441 6442
            threadInfo *pThreadInfo = infos + seq;
            pThreadInfo->threadID = seq;
            pThreadInfo->querySeq = i;
6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457

            if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {

                char sqlStr[MAX_TB_NAME_SIZE*2];
                sprintf(sqlStr, "use %s", g_queryInfo.dbName);
                verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
                if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) {
                    taos_close(taos);
                    free(infos);
                    free(pids);
                    errorPrint( "use database %s failed!\n\n",
                        g_queryInfo.dbName);
                    return -1;
                }
            }
6458

6459
            pThreadInfo->taos = NULL;// TODO: workaround to use separate taos connection;
6460

6461
            pthread_create(pids + seq, NULL, specifiedTableQuery,
6462
                pThreadInfo);
6463
        }
6464
    }
6465
  } else {
6466
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
6467
  }
6468

6469 6470
  taos_close(taos);

6471 6472 6473
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
6474 6475 6476 6477
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
    pidsOfSub  = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t));
    infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo));
6478 6479

    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
6480 6481
      free(infos);
      free(pids);
6482

6483
      ERROR_EXIT("memory allocation failed for create threads\n");
6484
    }
6485

6486
    int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
6487
    int threads = g_queryInfo.superQueryInfo.threadCnt;
6488

6489
    int64_t a = ntables / threads;
6490 6491 6492 6493
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
6494

6495
    int64_t b = 0;
6496 6497 6498
    if (threads != 0) {
      b = ntables % threads;
    }
6499

6500
    uint64_t startFrom = 0;
6501
    for (int i = 0; i < threads; i++) {
6502 6503
      threadInfo *pThreadInfo = infosOfSub + i;
      pThreadInfo->threadID = i;
6504

6505 6506 6507 6508 6509 6510
      pThreadInfo->start_table_from = startFrom;
      pThreadInfo->ntables = i<b?a+1:a;
      pThreadInfo->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = pThreadInfo->end_table_to + 1;
      pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection;
      pthread_create(pidsOfSub + i, NULL, superTableQuery, pThreadInfo);
6511 6512
    }

6513
    g_queryInfo.superQueryInfo.threadCnt = threads;
6514
  } else {
6515
    g_queryInfo.superQueryInfo.threadCnt = 0;
6516
  }
6517

6518 6519 6520 6521 6522 6523
  if ((nSqlCount > 0) && (nConcurrent > 0)) {
    for (int i = 0; i < nConcurrent; i++) {
      for (int j = 0; j < nSqlCount; j++) {
        pthread_join(pids[i * nSqlCount + j], NULL);
      }
    }
6524
  }
S
Shuaiqiang Chang 已提交
6525

6526
  tmfree((char*)pids);
6527
  tmfree((char*)infos);
6528

6529
  for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
6530
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
6531
  }
H
Hui Li 已提交
6532

6533
  tmfree((char*)pidsOfSub);
6534
  tmfree((char*)infosOfSub);
6535

6536
//  taos_close(taos);// TODO: workaround to use separate taos connection;
6537
  uint64_t endTs = taosGetTimestampMs();
6538

6539
  uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried +
6540 6541
    g_queryInfo.superQueryInfo.totalQueried;

6542
  fprintf(stderr, "==== completed total queries: %"PRIu64", the QPS of all threads: %10.3f====\n",
6543
          totalQueried,
6544
          (double)(totalQueried/((endTs-startTs)/1000.0)));
6545 6546 6547
  return 0;
}

6548 6549 6550 6551 6552 6553 6554 6555 6556
static void stable_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
  if (res == NULL || taos_errno(res) != 0) {
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
    return;
  }

  if (param)
6557
    fetchResult(res, (threadInfo *)param);
6558 6559 6560 6561 6562
  // tao_unscribe() will free result.
}

static void specified_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
6563
  if (res == NULL || taos_errno(res) != 0) {
6564 6565
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
6566 6567
    return;
  }
6568

6569
  if (param)
6570
    fetchResult(res, (threadInfo *)param);
6571
  // tao_unscribe() will free result.
H
hzcheng 已提交
6572 6573
}

6574
static TAOS_SUB* subscribeImpl(
6575
        QUERY_CLASS class,
6576
        threadInfo *pThreadInfo,
6577
        char *sql, char* topic, bool restart, uint64_t interval)
6578
{
6579
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
6580

6581 6582
  if ((SPECIFIED_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) {
6583 6584
    tsub = taos_subscribe(
            pThreadInfo->taos,
6585
            restart,
6586
            topic, sql, specified_sub_callback, (void*)pThreadInfo,
6587
            g_queryInfo.specifiedQueryInfo.subscribeInterval);
6588 6589
  } else if ((STABLE_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) {
6590 6591 6592 6593 6594
    tsub = taos_subscribe(
            pThreadInfo->taos,
            restart,
            topic, sql, stable_sub_callback, (void*)pThreadInfo,
            g_queryInfo.superQueryInfo.subscribeInterval);
6595
  } else {
6596 6597
    tsub = taos_subscribe(
            pThreadInfo->taos,
6598
            restart,
6599
            topic, sql, NULL, NULL, interval);
6600
  }
6601

6602 6603 6604
  if (tsub == NULL) {
    printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
    return NULL;
6605
  }
6606

6607 6608
  return tsub;
}
H
hzcheng 已提交
6609

6610 6611
static void *superSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6612
  char subSqlstr[MAX_QUERY_SQL_LENGTH];
6613
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT] = {0};
6614
  uint64_t tsubSeq;
H
hzcheng 已提交
6615

6616 6617
  if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
      errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n",
6618 6619 6620 6621 6622
              pThreadInfo->ntables,
              MAX_QUERY_SQL_COUNT);
      exit(-1);
  }

6623
  if (pThreadInfo->taos == NULL) {
6624
    pThreadInfo->taos = taos_connect(g_queryInfo.host,
6625 6626 6627 6628
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
6629
    if (pThreadInfo->taos == NULL) {
6630
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6631
            pThreadInfo->threadID, taos_errstr(NULL));
6632 6633 6634 6635
      return NULL;
    }
  }

6636 6637
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6638 6639
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6640 6641
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
6642 6643
    return NULL;
  }
6644

6645 6646 6647
  char topic[32] = {0};
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6648 6649

      tsubSeq = i - pThreadInfo->start_table_from;
6650 6651 6652 6653 6654
      verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n",
              __func__, __LINE__,
              pThreadInfo->threadID,
              pThreadInfo->start_table_from,
              pThreadInfo->end_table_to, i);
6655 6656
      sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"",
              i, pThreadInfo->querySeq);
6657
      memset(subSqlstr, 0, sizeof(subSqlstr));
6658 6659 6660 6661
      replaceChildTblName(
              g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq],
              subSqlstr, i);
      if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6662
        sprintf(pThreadInfo->filePath, "%s-%d",
6663 6664 6665 6666 6667 6668
                g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
      }

      debugPrint("%s() LN%d, [%d] subSqlstr: %s\n",
              __func__, __LINE__, pThreadInfo->threadID, subSqlstr);
6669
      tsub[tsubSeq] = subscribeImpl(
6670
              STABLE_CLASS,
6671
              pThreadInfo, subSqlstr, topic,
6672 6673
              g_queryInfo.superQueryInfo.subscribeRestart,
              g_queryInfo.superQueryInfo.subscribeInterval);
6674
      if (NULL == tsub[tsubSeq]) {
6675
        taos_close(pThreadInfo->taos);
6676 6677
        return NULL;
      }
6678
  }
6679 6680

  // start loop to consume result
6681 6682 6683 6684
  int consumed[MAX_QUERY_SQL_COUNT];
  for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) {
    consumed[i] = 0;
  }
6685
  TAOS_RES* res = NULL;
6686

6687 6688
  uint64_t st = 0, et = 0;

6689
  while(1) {
6690 6691
    for (uint64_t i = pThreadInfo->start_table_from;
            i <= pThreadInfo->end_table_to; i++) {
6692 6693 6694 6695
      tsubSeq = i - pThreadInfo->start_table_from;
      if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
          continue;
      }
6696

6697 6698
      st = taosGetTimestampMs();
      performancePrint("st: %"PRIu64" et: %"PRIu64" st-et: %"PRIu64"\n", st, et, (st - et));
6699
      res = taos_consume(tsub[tsubSeq]);
6700 6701 6702
      et = taosGetTimestampMs();
      performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st));

6703 6704
      if (res) {
          if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6705
              sprintf(pThreadInfo->filePath, "%s-%d",
6706 6707
                      g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
6708
              fetchResult(res, pThreadInfo);
6709 6710
          }
          if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6711
              sprintf(pThreadInfo->filePath, "%s-%d",
6712 6713
                      g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
6714
              fetchResult(res, pThreadInfo);
6715 6716 6717 6718 6719
          }
          consumed[tsubSeq] ++;

          if ((g_queryInfo.superQueryInfo.subscribeKeepProgress)
                  && (consumed[tsubSeq] >=
6720
                      g_queryInfo.superQueryInfo.resubAfterConsume)) {
6721 6722 6723 6724
              printf("keepProgress:%d, resub super table query: %"PRIu64"\n",
                      g_queryInfo.superQueryInfo.subscribeKeepProgress,
                      pThreadInfo->querySeq);
              taos_unsubscribe(tsub[tsubSeq],
6725
                    g_queryInfo.superQueryInfo.subscribeKeepProgress);
6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738
              consumed[tsubSeq]= 0;
              tsub[tsubSeq] = subscribeImpl(
                      STABLE_CLASS,
                      pThreadInfo, subSqlstr, topic,
                      g_queryInfo.superQueryInfo.subscribeRestart,
                      g_queryInfo.superQueryInfo.subscribeInterval
                      );
              if (NULL == tsub[tsubSeq]) {
                  taos_close(pThreadInfo->taos);
                  return NULL;
              }
          }
      }
H
hzcheng 已提交
6739 6740
    }
  }
6741
  taos_free_result(res);
6742

6743 6744
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6745 6746
    tsubSeq = i - pThreadInfo->start_table_from;
    taos_unsubscribe(tsub[tsubSeq], 0);
6747
  }
6748

6749
  taos_close(pThreadInfo->taos);
H
hzcheng 已提交
6750 6751 6752
  return NULL;
}

6753 6754
static void *specifiedSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6755
//  TAOS_SUB*  tsub = NULL;
6756

6757
  if (pThreadInfo->taos == NULL) {
6758
    pThreadInfo->taos = taos_connect(g_queryInfo.host,
6759 6760 6761 6762
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
6763
    if (pThreadInfo->taos == NULL) {
6764
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6765
            pThreadInfo->threadID, taos_errstr(NULL));
6766 6767 6768 6769
      return NULL;
    }
  }

6770 6771
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6772
  debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
6773 6774
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6775 6776
    return NULL;
  }
6777

6778 6779 6780 6781 6782 6783
  sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
          "taosdemo-subscribe-%"PRIu64"-%d",
          pThreadInfo->querySeq,
          pThreadInfo->threadID);
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq] != NULL) {
      sprintf(pThreadInfo->filePath, "%s-%d",
6784
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
6785
                pThreadInfo->threadID);
6786
  }
6787 6788
  g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl(
          SPECIFIED_CLASS, pThreadInfo,
6789
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
6790
          g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
6791 6792
          g_queryInfo.specifiedQueryInfo.subscribeRestart,
          g_queryInfo.specifiedQueryInfo.subscribeInterval);
6793
  if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) {
6794 6795
      taos_close(pThreadInfo->taos);
      return NULL;
6796
  }
6797

6798
  // start loop to consume result
6799

6800
  g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0;
6801
  while(1) {
6802
      if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) {
6803 6804
        continue;
      }
6805

6806 6807 6808
      g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] = taos_consume(
              g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]);
      if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) {
6809
          if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6810
              sprintf(pThreadInfo->filePath, "%s-%d",
6811 6812
                      g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
6813
              fetchResult(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], pThreadInfo);
6814 6815
          }

6816
          g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] ++;
6817
          if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress)
6818
                && (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] >=
6819 6820 6821 6822
                    g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
              printf("keepProgress:%d, resub specified query: %"PRIu64"\n",
                    g_queryInfo.specifiedQueryInfo.subscribeKeepProgress,
                    pThreadInfo->querySeq);
6823 6824
              g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0;
              taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID],
6825
                      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
6826
              g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl(
6827 6828 6829
                      SPECIFIED_CLASS,
                      pThreadInfo,
                      g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
6830
                      g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
6831 6832
                      g_queryInfo.specifiedQueryInfo.subscribeRestart,
                      g_queryInfo.specifiedQueryInfo.subscribeInterval);
6833
              if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) {
6834 6835
                taos_close(pThreadInfo->taos);
                return NULL;
6836 6837
              }
          }
6838 6839
      }
  }
6840 6841
  taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]);
  taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->querySeq], 0);
6842
  taos_close(pThreadInfo->taos);
6843

H
hzcheng 已提交
6844 6845 6846
  return NULL;
}

6847
static int subscribeTestProcess() {
6848
  setupForAnsiEscape();
6849
  printfQueryMeta();
6850
  resetAfterAnsiEscape();
6851

6852
  prompt();
6853

6854 6855 6856 6857 6858
  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
6859
          g_queryInfo.port);
6860
  if (taos == NULL) {
6861 6862
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6863 6864 6865
    exit(-1);
  }

6866
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6867 6868
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6869 6870 6871
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6872 6873
  }

6874 6875
  taos_close(taos); // TODO: workaround to use separate taos connection;

6876 6877
  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
6878 6879 6880 6881 6882

  pthread_t  *pidsOfStable  = NULL;
  threadInfo *infosOfStable = NULL;

  //==== create threads for query for specified table
6883
  if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) {
6884
    debugPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n",
6885
              __func__, __LINE__,
6886 6887 6888 6889 6890 6891 6892 6893
              g_queryInfo.specifiedQueryInfo.sqlCount);
  } else {
    if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) {
        errorPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n",
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount);
        exit(-1);
    }
6894

6895
    pids  = malloc(
6896 6897 6898
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(pthread_t));
6899
    infos = malloc(
6900 6901 6902
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(threadInfo));
6903 6904 6905 6906
    if ((NULL == pids) || (NULL == infos)) {
        errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
        exit(-1);
    }
6907

6908 6909 6910
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
            uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
6911 6912 6913 6914 6915
            threadInfo *pThreadInfo = infos + seq;
            pThreadInfo->threadID = seq;
            pThreadInfo->querySeq = i;
            pThreadInfo->taos = NULL;  // TODO: workaround to use separate taos connection;
            pthread_create(pids + seq, NULL, specifiedSubscribe, pThreadInfo);
6916
        }
6917
    }
H
hzcheng 已提交
6918
  }
6919

6920
  //==== create threads for super table query
6921
  if (g_queryInfo.superQueryInfo.sqlCount <= 0) {
6922
    debugPrint("%s() LN%d, super table query sqlCount %"PRIu64".\n",
6923
              __func__, __LINE__,
6924
              g_queryInfo.superQueryInfo.sqlCount);
6925 6926
  } else {
    if ((g_queryInfo.superQueryInfo.sqlCount > 0)
6927
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
6928 6929 6930
        pidsOfStable  = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
6931
            sizeof(pthread_t));
6932 6933 6934
        infosOfStable = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
6935
            sizeof(threadInfo));
6936 6937
        if ((NULL == pidsOfStable) || (NULL == infosOfStable)) {
            errorPrint("%s() LN%d, malloc failed for create threads\n",
6938
              __func__, __LINE__);
6939 6940 6941
            // taos_close(taos);
            exit(-1);
        }
6942

6943 6944
        int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
        int threads = g_queryInfo.superQueryInfo.threadCnt;
6945

6946 6947 6948 6949 6950
        int64_t a = ntables / threads;
        if (a < 1) {
            threads = ntables;
            a = 1;
        }
6951

6952 6953 6954 6955
        int64_t b = 0;
        if (threads != 0) {
            b = ntables % threads;
        }
6956

6957
        for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
6958
            uint64_t startFrom = 0;
6959 6960
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
6961 6962 6963 6964 6965 6966 6967 6968 6969
                threadInfo *pThreadInfo = infosOfStable + seq;
                pThreadInfo->threadID = seq;
                pThreadInfo->querySeq = i;

                pThreadInfo->start_table_from = startFrom;
                pThreadInfo->ntables = j<b?a+1:a;
                pThreadInfo->end_table_to = j<b?startFrom+a:startFrom+a-1;
                startFrom = pThreadInfo->end_table_to + 1;
                pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection;
6970
                pthread_create(pidsOfStable + seq,
6971
                        NULL, superSubscribe, pThreadInfo);
6972 6973
            }
        }
6974

6975
        g_queryInfo.superQueryInfo.threadCnt = threads;
6976

6977 6978 6979 6980 6981 6982
        for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                pthread_join(pidsOfStable[seq], NULL);
            }
        }
6983
    }
H
hzcheng 已提交
6984
  }
6985

6986 6987 6988 6989 6990
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
    for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
        uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
        pthread_join(pids[seq], NULL);
    }
6991
  }
H
hzcheng 已提交
6992

6993
  tmfree((char*)pids);
6994
  tmfree((char*)infos);
H
hzcheng 已提交
6995

6996 6997
  tmfree((char*)pidsOfStable);
  tmfree((char*)infosOfStable);
6998
//   taos_close(taos);
6999
  return 0;
H
hzcheng 已提交
7000 7001
}

7002
static void initOfInsertMeta() {
7003
  memset(&g_Dbs, 0, sizeof(SDbs));
7004

7005
  // set default values
7006
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7007
  g_Dbs.port = 6030;
7008 7009
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
7010
  g_Dbs.threadCount = 2;
7011 7012

  g_Dbs.use_metric = g_args.use_metric;
L
Liu Tao 已提交
7013 7014
}

7015
static void initOfQueryMeta() {
7016
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
7017

7018
  // set default values
7019
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7020
  g_queryInfo.port = 6030;
7021 7022
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
L
Liu Tao 已提交
7023 7024
}

7025
static void setParaFromArg(){
7026
  if (g_args.host) {
7027
    tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE);
7028
  } else {
7029
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7030
  }
L
Liu Tao 已提交
7031

7032
  if (g_args.user) {
7033
    tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE);
7034
  }
7035 7036

  if (g_args.password) {
7037
    tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE);
7038 7039
  }

7040 7041
  if (g_args.port) {
    g_Dbs.port = g_args.port;
7042
  }
L
Liu Tao 已提交
7043

7044 7045 7046
  g_Dbs.threadCount = g_args.num_of_threads;
  g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;

7047 7048
  g_Dbs.dbCount = 1;
  g_Dbs.db[0].drop = 1;
7049

7050
  tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE);
7051
  g_Dbs.db[0].dbCfg.replica = g_args.replica;
7052
  tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
7053

7054
  tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
7055 7056 7057 7058 7059

  g_Dbs.use_metric = g_args.use_metric;
  g_Dbs.insert_only = g_args.insert_only;

  g_Dbs.do_aggreFunc = true;
L
Liu Tao 已提交
7060

7061 7062
  char dataString[STRING_LEN];
  char **data_type = g_args.datatype;
7063

7064
  memset(dataString, 0, STRING_LEN);
L
Liu Tao 已提交
7065

7066 7067
  if (strcasecmp(data_type[0], "BINARY") == 0
          || strcasecmp(data_type[0], "BOOL") == 0
7068
          || strcasecmp(data_type[0], "NCHAR") == 0 ) {
7069
    g_Dbs.do_aggreFunc = false;
L
Liu Tao 已提交
7070
  }
H
hzcheng 已提交
7071

7072 7073 7074 7075 7076
  if (g_args.use_metric) {
    g_Dbs.db[0].superTblCount = 1;
    tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE);
    g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
    g_Dbs.threadCount = g_args.num_of_threads;
7077
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
7078
    g_Dbs.asyncMode = g_args.async_mode;
7079

7080 7081 7082 7083
    g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
    g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
    g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
7084
    tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix,
7085 7086 7087
            g_args.tb_prefix, MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE);
7088
    tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp,
7089
            "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
7090
    g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
7091

7092
    g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
7093
    g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len;
7094

7095 7096 7097 7098 7099
    g_Dbs.db[0].superTbls[0].columnCount = 0;
    for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
      if (data_type[i] == NULL) {
        break;
      }
7100

7101
      tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
7102
              data_type[i], MAX_TB_NAME_SIZE);
7103
      g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;
7104 7105
      g_Dbs.db[0].superTbls[0].columnCount++;
    }
7106

7107 7108 7109
    if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
      g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
    } else {
7110 7111 7112 7113
      for (int i = g_Dbs.db[0].superTbls[0].columnCount;
              i < g_args.num_of_CPR; i++) {
        tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
                "INT", MAX_TB_NAME_SIZE);
7114
        g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;
7115 7116 7117
        g_Dbs.db[0].superTbls[0].columnCount++;
      }
    }
7118

7119 7120
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType,
            "INT", MAX_TB_NAME_SIZE);
7121
    g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;
7122

7123 7124
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType,
            "BINARY", MAX_TB_NAME_SIZE);
7125 7126
    g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;
    g_Dbs.db[0].superTbls[0].tagCount = 2;
7127
  } else {
7128 7129
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
    g_Dbs.db[0].superTbls[0].tagCount = 0;
H
hzcheng 已提交
7130
  }
7131 7132 7133 7134 7135 7136
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
7137

7138 7139 7140
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
7141 7142 7143
    exit(-1);
  }

7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

7168
static void querySqlFile(TAOS* taos, char* sqlFile)
7169 7170 7171 7172 7173 7174
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
7175

7176 7177 7178 7179 7180 7181
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

7182
  double t = taosGetTimestampMs();
7183

7184
  while((read_len = tgetline(&line, &line_len, fp)) != -1) {
7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
7197
    }
7198 7199

    memcpy(cmd + cmd_len, line, read_len);
7200
    verbosePrint("%s() LN%d cmd: %s\n", __func__, __LINE__, cmd);
7201 7202 7203
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) {
        errorPrint("%s() LN%d, queryDbExec %s failed!\n",
               __func__, __LINE__, cmd);
7204 7205 7206 7207 7208
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
7209 7210
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
7211 7212
  }

7213
  t = taosGetTimestampMs() - t;
7214
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
7215

7216 7217 7218 7219
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
7220 7221
}

7222
static void testMetaFile() {
7223
    if (INSERT_TEST == g_args.test_mode) {
7224 7225
      if (g_Dbs.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
7226

7227
      insertTestProcess();
7228

7229
    } else if (QUERY_TEST == g_args.test_mode) {
7230 7231
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7232

7233
      queryTestProcess();
7234

7235
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
7236 7237
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7238

7239
      subscribeTestProcess();
7240

7241 7242 7243
    }  else {
      ;
    }
7244
}
7245

7246
static void queryResult() {
7247
  // query data
7248

7249
  pthread_t read_id;
7250 7251 7252 7253
  threadInfo *pThreadInfo = malloc(sizeof(threadInfo));
  assert(pThreadInfo);
  pThreadInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
  pThreadInfo->start_table_from = 0;
7254

7255
  //pThreadInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
7256
  if (g_args.use_metric) {
7257 7258 7259 7260
    pThreadInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
    pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
    pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
    tstrncpy(pThreadInfo->tb_prefix,
7261 7262
          g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE);
  } else {
7263 7264 7265
    pThreadInfo->ntables = g_args.num_of_tables;
    pThreadInfo->end_table_to = g_args.num_of_tables -1;
    tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE);
7266 7267
  }

7268
  pThreadInfo->taos = taos_connect(
7269 7270 7271 7272 7273
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
          g_Dbs.port);
7274
  if (pThreadInfo->taos == NULL) {
7275 7276
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
7277
    free(pThreadInfo);
7278 7279 7280
    exit(-1);
  }

7281
  tstrncpy(pThreadInfo->filePath, g_Dbs.resultFile, MAX_FILE_NAME_LEN);
7282 7283

  if (!g_Dbs.use_metric) {
7284
    pthread_create(&read_id, NULL, readTable, pThreadInfo);
7285
  } else {
7286
    pthread_create(&read_id, NULL, readMetric, pThreadInfo);
7287 7288
  }
  pthread_join(read_id, NULL);
7289 7290
  taos_close(pThreadInfo->taos);
  free(pThreadInfo);
7291 7292
}

7293 7294
static void testCmdLine() {

7295 7296 7297 7298 7299 7300 7301 7302 7303 7304
  if (strlen(configDir)) {
    wordexp_t full_path;
    if (wordexp(configDir, &full_path, 0) != 0) {
      errorPrint( "Invalid path %s\n", configDir);
      return;
    }
    taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
    wordfree(&full_path);
  }

7305 7306
  g_args.test_mode = INSERT_TEST;
  insertTestProcess();
7307

7308
  if (false == g_Dbs.insert_only)
7309
    queryResult();
7310 7311
}

7312 7313 7314
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

7315
  debugPrint("meta file: %s\n", g_args.metaFile);
7316 7317 7318

  if (g_args.metaFile) {
    initOfInsertMeta();
7319
    initOfQueryMeta();
7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
7333 7334 7335 7336
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
7337
          g_Dbs.port);
7338
      querySqlFile(qtaos, g_args.sqlFile);
7339 7340 7341 7342 7343
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
7344 7345 7346

    if (g_dupstr)
        free(g_dupstr);
H
hzcheng 已提交
7347
  }
7348 7349

  return 0;
H
hzcheng 已提交
7350
}
7351