taosdemo.c 245.9 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

21
#include <stdint.h>
S
slguan 已提交
22
#define _GNU_SOURCE
23 24
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
41
#else
S
TD-1057  
Shengliang Guan 已提交
42 43
  #include <regex.h>
  #include <stdio.h>
44
#endif
H
hzcheng 已提交
45

46
#include <assert.h>
47
#include <stdlib.h>
48 49
#include "cJSON.h"

50
#include "os.h"
H
hzcheng 已提交
51
#include "taos.h"
52
#include "taoserror.h"
H
Hui Li 已提交
53
#include "tutil.h"
S
slguan 已提交
54

55 56 57
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
58 59
extern char configDir[];

60 61 62 63
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

64 65 66 67 68 69
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};
70

71 72
#define MAX_RECORDS_PER_REQ     32766

73 74
#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
75
#define COND_BUF_LEN        BUFFER_SIZE - 30
76 77
#define MAX_USERNAME_SIZE  64
#define MAX_PASSWORD_SIZE  64
78
#define MAX_DB_NAME_SIZE   64
79
#define MAX_HOSTNAME_SIZE  64
80
#define MAX_TB_NAME_SIZE   64
81
#define MAX_DATA_SIZE      (16*1024)+20     // max record len: 16*1024, timestamp string and ,('') need extra space
82 83 84 85 86 87 88 89 90 91
#define MAX_NUM_DATATYPE   10
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
#define MAX_FILE_NAME_LEN  256

#define   MAX_SAMPLES_ONCE_FROM_FILE   10000
#define   MAX_NUM_DATATYPE 10

#define   MAX_DB_COUNT           8
92
#define   MAX_SUPER_TABLE_COUNT  200
93 94 95
#define   MAX_COLUMN_COUNT       1024
#define   MAX_TAG_COUNT          128

96
#define   MAX_QUERY_SQL_COUNT    100
97
#define   MAX_QUERY_SQL_LENGTH   1024
98 99

#define   MAX_DATABASE_COUNT     256
100
#define INPUT_BUF_LEN   256
101

102
#define DEFAULT_TIMESTAMP_STEP  1
103

104

105 106 107 108 109
typedef enum CREATE_SUB_TALBE_MOD_EN {
  PRE_CREATE_SUBTBL,
  AUTO_CREATE_SUBTBL,
  NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
110

111 112
typedef enum TALBE_EXISTS_EN {
  TBL_NO_EXISTS,
113
  TBL_ALREADY_EXISTS,
114 115 116
  TBL_EXISTS_BUTT
} TALBE_EXISTS_EN;

117
enum enumSYNC_MODE {
118 119
  SYNC_MODE,
  ASYNC_MODE,
120 121
  MODE_BUT
};
122

123 124 125 126 127 128
typedef enum enumQUERY_CLASS {
    SPECIFIED_CLASS,
    STABLE_CLASS,
    CLASS_BUT
} QUERY_CLASS;

129 130 131 132 133 134
typedef enum enum_INSERT_MODE {
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
} INSERT_MODE;

135
typedef enum enumQUERY_TYPE {
136
  NO_INSERT_TYPE,
137
  INSERT_TYPE,
138
  QUERY_TYPE_BUT
139
} QUERY_TYPE;
140 141 142 143 144 145 146

enum _show_db_index {
  TSDB_SHOW_DB_NAME_INDEX,
  TSDB_SHOW_DB_CREATED_TIME_INDEX,
  TSDB_SHOW_DB_NTABLES_INDEX,
  TSDB_SHOW_DB_VGROUPS_INDEX,
  TSDB_SHOW_DB_REPLICA_INDEX,
147
  TSDB_SHOW_DB_QUORUM_INDEX,
148 149 150 151 152 153 154 155 156 157
  TSDB_SHOW_DB_DAYS_INDEX,
  TSDB_SHOW_DB_KEEP_INDEX,
  TSDB_SHOW_DB_CACHE_INDEX,
  TSDB_SHOW_DB_BLOCKS_INDEX,
  TSDB_SHOW_DB_MINROWS_INDEX,
  TSDB_SHOW_DB_MAXROWS_INDEX,
  TSDB_SHOW_DB_WALLEVEL_INDEX,
  TSDB_SHOW_DB_FSYNC_INDEX,
  TSDB_SHOW_DB_COMP_INDEX,
  TSDB_SHOW_DB_CACHELAST_INDEX,
158
  TSDB_SHOW_DB_PRECISION_INDEX,
159 160 161 162 163 164 165 166 167 168
  TSDB_SHOW_DB_UPDATE_INDEX,
  TSDB_SHOW_DB_STATUS_INDEX,
  TSDB_MAX_SHOW_DB
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
  TSDB_SHOW_STABLES_NAME_INDEX,
  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
  TSDB_SHOW_STABLES_COLUMNS_INDEX,
169 170
  TSDB_SHOW_STABLES_METRIC_INDEX,
  TSDB_SHOW_STABLES_UID_INDEX,
171
  TSDB_SHOW_STABLES_TID_INDEX,
172
  TSDB_SHOW_STABLES_VGID_INDEX,
173
  TSDB_MAX_SHOW_STABLES
174 175
};

176 177 178 179 180 181 182 183 184 185 186 187 188 189
enum _describe_table_index {
  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
  TSDB_MAX_DESCRIBE_METRIC
};

typedef struct {
  char field[TSDB_COL_NAME_LEN + 1];
  char type[16];
  int length;
  char note[128];
} SColDes;
H
hzcheng 已提交
190 191

/* Used by main to communicate with parse_opt. */
192 193
static char *g_dupstr = NULL;

194 195
typedef struct SArguments_S {
  char *   metaFile;
196
  uint32_t test_mode;
197 198 199 200 201 202 203 204 205
  char *   host;
  uint16_t port;
  char *   user;
  char *   password;
  char *   database;
  int      replica;
  char *   tb_prefix;
  char *   sqlFile;
  bool     use_metric;
206
  bool     drop_database;
207
  bool     insert_only;
208
  bool     answer_yes;
209
  bool     debug_print;
210
  bool     verbose_print;
211
  bool     performance_print;
212
  char *   output_file;
213
  bool     async_mode;
214
  char *   datatype[MAX_NUM_DATATYPE + 1];
215 216 217 218
  uint32_t len_of_binary;
  uint32_t num_of_CPR;
  uint32_t num_of_threads;
  uint64_t insert_interval;
219
  int64_t  query_times;
220 221 222
  uint64_t interlace_rows;
  uint64_t num_of_RPR;                  // num_of_records_per_req
  uint64_t max_sql_len;
223 224
  int64_t  num_of_tables;
  int64_t  num_of_DPT;
225
  int      abort;
226 227
  int      disorderRatio;               // 0: no disorder, >0: x%
  int      disorderRange;               // ms or us by database precision
228
  uint32_t method_of_delete;
229
  char **  arg_list;
230 231
  uint64_t totalInsertRows;
  uint64_t totalAffectedRows;
232 233 234
} SArguments;

typedef struct SColumn_S {
235 236 237
  char      field[TSDB_COL_NAME_LEN + 1];
  char      dataType[MAX_TB_NAME_SIZE];
  uint32_t  dataLen;
238
  char      note[128];
239 240 241
} StrColumn;

typedef struct SSuperTable_S {
242
  char         sTblName[MAX_TB_NAME_SIZE+1];
243
  int64_t      childTblCount;
244
  bool         childTblExists;          // 0: no, 1: yes
245 246
  uint64_t     batchCreateTableNum;     // 0: no batch,  > 0: batch table number in one sql
  uint8_t      autoCreateTable;         // 0: create sub table, 1: auto create sub table
247
  char         childTblPrefix[MAX_TB_NAME_SIZE];
248
  char         dataSource[MAX_TB_NAME_SIZE+1];  // rand_gen or sample
249
  char         insertMode[MAX_TB_NAME_SIZE];    // taosc, rest
250
  int64_t      childTblLimit;
251
  uint64_t     childTblOffset;
252

253
//  int          multiThreadWriteOneTbl;  // 0: no, 1: yes
254
  uint64_t     interlaceRows;           //
255 256
  int          disorderRatio;           // 0: no disorder, >0: x%
  int          disorderRange;           // ms or us by database precision
257
  uint64_t     maxSqlLen;               //
258

259
  uint64_t     insertInterval;          // insert interval, will override global insert interval
260
  int64_t      insertRows;
261
  int64_t      timeStampStep;
262
  char         startTimestamp[MAX_TB_NAME_SIZE];
263
  char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
264 265
  char         sampleFile[MAX_FILE_NAME_LEN+1];
  char         tagsFile[MAX_FILE_NAME_LEN+1];
266

267
  uint32_t     columnCount;
268
  StrColumn    columns[MAX_COLUMN_COUNT];
269
  uint32_t     tagCount;
270 271 272
  StrColumn    tags[MAX_TAG_COUNT];

  char*        childTblName;
273
  char*        colsOfCreateChildTable;
274 275
  uint64_t     lenOfOneRow;
  uint64_t     lenOfTagOfOneRow;
276 277 278 279 280

  char*        sampleDataBuf;
  //int          sampleRowCount;
  //int          sampleUsePos;

281
  uint32_t     tagSource;    // 0: rand, 1: tag sample
282
  char*        tagDataBuf;
283 284
  uint32_t     tagSampleCount;
  uint32_t     tagUsePos;
285

286
  // statistics
287 288
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
289 290 291 292 293
} SSuperTable;

typedef struct {
  char     name[TSDB_DB_NAME_LEN + 1];
  char     create_time[32];
294
  int64_t  ntables;
295
  int32_t  vgroups;
296 297
  int16_t  replica;
  int16_t  quorum;
298
  int16_t  days;
299 300 301 302 303 304 305 306 307 308 309 310 311 312
  char     keeplist[32];
  int32_t  cache; //MB
  int32_t  blocks;
  int32_t  minrows;
  int32_t  maxrows;
  int8_t   wallevel;
  int32_t  fsync;
  int8_t   comp;
  int8_t   cachelast;
  char     precision[8];   // time resolution
  int8_t   update;
  char     status[16];
} SDbInfo;

313
typedef struct SDbCfg_S {
314
//  int       maxtablesPerVnode;
315 316
  uint32_t  minRows;        // 0 means default
  uint32_t  maxRows;        // 0 means default
317 318
  int       comp;
  int       walLevel;
319
  int       cacheLast;
320
  int       fsync;
321 322 323 324 325 326 327
  int       replica;
  int       update;
  int       keep;
  int       days;
  int       cache;
  int       blocks;
  int       quorum;
328
  char      precision[MAX_TB_NAME_SIZE];
329 330 331 332
} SDbCfg;

typedef struct SDataBase_S {
  char         dbName[MAX_DB_NAME_SIZE];
333
  bool         drop;  // 0: use exists, 1: if exists, drop then new create
334
  SDbCfg       dbCfg;
335
  uint64_t     superTblCount;
336 337 338 339
  SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
} SDataBase;

typedef struct SDbs_S {
340
  char         cfgDir[MAX_FILE_NAME_LEN+1];
341
  char         host[MAX_HOSTNAME_SIZE];
342 343
  struct sockaddr_in serv_addr;

344
  uint16_t     port;
345 346
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
347
  char         resultFile[MAX_FILE_NAME_LEN+1];
348 349 350
  bool         use_metric;
  bool         insert_only;
  bool         do_aggreFunc;
351
  bool         asyncMode;
352

353 354 355
  uint32_t     threadCount;
  uint32_t     threadCountByCreateTbl;
  uint32_t     dbCount;
356 357 358
  SDataBase    db[MAX_DB_COUNT];

  // statistics
359 360
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
361

362 363
} SDbs;

364
typedef struct SpecifiedQueryInfo_S {
365
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
366
  uint32_t     concurrent;
367
  int          sqlCount;
368
  uint32_t     asyncMode; // 0: sync, 1: async
369 370
  uint64_t     subscribeInterval; // ms
  uint64_t     queryTimes;
371
  bool         subscribeRestart;
372
  int          subscribeKeepProgress;
373
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
374
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
375
  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
376
  int          endAfterConsume[MAX_QUERY_SQL_COUNT];
377
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
378 379 380
  char         topic[MAX_QUERY_SQL_COUNT][32];
  int          consumed[MAX_QUERY_SQL_COUNT];
  TAOS_RES*    res[MAX_QUERY_SQL_COUNT];
381
  uint64_t     totalQueried;
382
} SpecifiedQueryInfo;
383

384
typedef struct SuperQueryInfo_S {
385
  char         sTblName[MAX_TB_NAME_SIZE+1];
386 387
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint32_t     threadCnt;
388
  uint32_t     asyncMode; // 0: sync, 1: async
389
  uint64_t     subscribeInterval; // ms
390
  bool         subscribeRestart;
391
  int          subscribeKeepProgress;
392
  uint64_t     queryTimes;
393
  int64_t      childTblCount;
394
  char         childTblPrefix[MAX_TB_NAME_SIZE];
395
  int          sqlCount;
396
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
397
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
398
  int          resubAfterConsume;
399
  int          endAfterConsume;
400
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
401

402
  char*        childTblName;
403
  uint64_t     totalQueried;
404
} SuperQueryInfo;
405 406

typedef struct SQueryMetaInfo_S {
407
  char         cfgDir[MAX_FILE_NAME_LEN+1];
408
  char         host[MAX_HOSTNAME_SIZE];
409
  uint16_t     port;
410
  struct       sockaddr_in serv_addr;
411 412
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
413
  char         dbName[MAX_DB_NAME_SIZE+1];
414
  char         queryMode[MAX_TB_NAME_SIZE];  // taosc, rest
415

416
  SpecifiedQueryInfo  specifiedQueryInfo;
417
  SuperQueryInfo      superQueryInfo;
418
  uint64_t     totalQueried;
419 420 421
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
422 423 424 425
  TAOS *    taos;
  int       threadID;
  char      db_name[MAX_DB_NAME_SIZE+1];
  uint32_t  time_precision;
426 427
  char      filePath[4096];
  FILE      *fp;
428 429 430
  char      tb_prefix[MAX_TB_NAME_SIZE];
  uint64_t  start_table_from;
  uint64_t  end_table_to;
431
  int64_t   ntables;
432 433 434 435
  uint64_t  data_of_rate;
  int64_t   start_time;
  char*     cols;
  bool      use_metric;
436 437 438
  SSuperTable* superTblInfo;

  // for async insert
439 440
  tsem_t    lock_sem;
  int64_t   counter;
441 442
  uint64_t  st;
  uint64_t  et;
443
  uint64_t  lastTs;
444

445
  // sample data
446
  int64_t   samplePos;
447
  // statistics
448 449
  uint64_t  totalInsertRows;
  uint64_t  totalAffectedRows;
450 451

  // insert delay statistics
452 453 454 455 456
  uint64_t  cntDelay;
  uint64_t  totalDelay;
  uint64_t  avgDelay;
  uint64_t  maxDelay;
  uint64_t  minDelay;
457

458
  // seq of query or subscribe
459
  uint64_t  querySeq;   // sequence number of sql command
460
  TAOS_SUB*  tsub;
461

462 463
} threadInfo;

464
#ifdef WINDOWS
465 466
#define _CRT_RAND_S

467
#include <windows.h>
468 469 470 471 472
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
473 474
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
475 476
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
477 478 479 480

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

481
static void setupForAnsiEscape(void) {
482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
500
  }
501 502
}

503
static void resetAfterAnsiEscape(void) {
504
  // Reset colors
505
  printf("\x1b[0m");
506 507 508 509 510 511

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
512

513
static int taosRandom()
514 515 516 517 518 519
{
    int number;
    rand_s(&number);

    return number;
}
520
#else   // Not windows
521
static void setupForAnsiEscape(void) {}
522

523
static void resetAfterAnsiEscape(void) {
524 525 526
  // Reset colors
  printf("\x1b[0m");
}
527

528 529
#include <time.h>

530
static int taosRandom()
531
{
532
  return rand();
533 534
}

535
#endif // ifdef Windows
536

537
static void prompt();
538
static int createDatabasesAndStables();
539
static void createChildTables();
540
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet);
541
static int postProceSql(char *host, struct sockaddr_in *pServAddr,
542
        uint16_t port, char* sqlstr, threadInfo *pThreadInfo);
543 544
static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq,
                  int disorderRatio, int disorderRange);
545 546 547 548 549 550 551

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
552
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)",
553 554
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

555
SArguments g_args = {
556 557
                     NULL,            // metaFile
                     0,               // test_mode
558 559 560
                     "127.0.0.1",     // host
                     6030,            // port
                     "root",          // user
561
                     #ifdef _TD_POWER_
562 563 564 565 566 567 568 569
                     "powerdb",      // password
                     #else
                     "taosdata",      // password
                     #endif
                     "test",          // database
                     1,               // replica
                     "t",             // tb_prefix
                     NULL,            // sqlFile
570
                     true,            // use_metric
571
                     true,            // drop_database
572
                     true,            // insert_only
573
                     false,           // debug_print
574
                     false,           // verbose_print
575
                     false,           // performance statistic print
576 577 578 579
                     false,           // answer_yes;
                     "./output.txt",  // output_file
                     0,               // mode : sync or async
                     {
580 581 582 583
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
584 585
                     },
                     16,              // len_of_binary
586
                     4,               // num_of_CPR
587 588
                     10,              // num_of_connections/thread
                     0,               // insert_interval
589
                     1,               // query_times
590
                     0,               // interlace_rows;
591
                     30000,           // num_of_RPR
592
                     (1024*1024),         // max_sql_len
593 594 595 596 597 598 599 600 601 602 603 604
                     10000,           // num_of_tables
                     10000,           // num_of_DPT
                     0,               // abort
                     0,               // disorderRatio
                     1000,            // disorderRange
                     1,               // method_of_delete
                     NULL             // arg_list
};



static SDbs            g_Dbs;
605
static int64_t         g_totalChildTables = 0;
606 607 608 609
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

#define debugPrint(fmt, ...) \
610
    do { if (g_args.debug_print || g_args.verbose_print) \
611
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
612

613
#define verbosePrint(fmt, ...) \
614 615
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
616

617 618 619 620
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)

621 622 623 624
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)


625
///////////////////////////////////////////////////
H
hzcheng 已提交
626

627 628
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

629 630 631 632 633 634 635 636
#ifndef TAOSDEMO_COMMIT_SHA1
#define TAOSDEMO_COMMIT_SHA1 "unknown"
#endif

#ifndef TD_VERNUMBER
#define TD_VERNUMBER    "unknown"
#endif

637
#ifndef TAOSDEMO_STATUS
638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654
#define TAOSDEMO_STATUS "unknown"
#endif

static void printVersion() {
    char tdengine_ver[] = TD_VERNUMBER;
    char taosdemo_ver[] = TAOSDEMO_COMMIT_SHA1;
    char taosdemo_status[] = TAOSDEMO_STATUS;

    if (strlen(taosdemo_status) == 0) {
        printf("taosdemo verison %s-%s\n",
                tdengine_ver, taosdemo_ver);
    } else {
        printf("taosdemo verison %s-%s, status:%s\n",
                tdengine_ver, taosdemo_ver, taosdemo_status);
    }
}

655
static void printHelp() {
656
  char indent[10] = "        ";
657
  printf("%s%s%s%s\n", indent, "-f", indent,
658
          "The meta file to the execution procedure. Default is './meta.json'.");
659
  printf("%s%s%s%s\n", indent, "-u", indent,
660 661
          "The TDengine user name to use when connecting to the server. Default is 'root'.");
#ifdef _TD_POWER_
662
  printf("%s%s%s%s\n", indent, "-P", indent,
663
          "The password to use when connecting to the server. Default is 'powerdb'.");
664
  printf("%s%s%s%s\n", indent, "-c", indent,
665 666
          "Configuration directory. Default is '/etc/power/'.");
#else
667
  printf("%s%s%s%s\n", indent, "-P", indent,
668
          "The password to use when connecting to the server. Default is 'taosdata'.");
669
  printf("%s%s%s%s\n", indent, "-c", indent,
670
          "Configuration directory. Default is '/etc/taos/'.");
671
#endif
672
  printf("%s%s%s%s\n", indent, "-h", indent,
673
          "The host to connect to TDengine. Default is localhost.");
674
  printf("%s%s%s%s\n", indent, "-p", indent,
675
          "The TCP/IP port number to use for the connection. Default is 0.");
676
  printf("%s%s%s%s\n", indent, "-d", indent,
677
          "Destination database. Default is 'test'.");
678
  printf("%s%s%s%s\n", indent, "-a", indent,
679
          "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
680
  printf("%s%s%s%s\n", indent, "-m", indent,
681 682
          "Table prefix name. Default is 't'.");
  printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
683
  printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
684
  printf("%s%s%s%s\n", indent, "-o", indent,
685
          "Direct output to the named file. Default is './output.txt'.");
686
  printf("%s%s%s%s\n", indent, "-q", indent,
687
          "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC.");
688
  printf("%s%s%s%s\n", indent, "-b", indent,
689
          "The data_type of columns, default: INT,INT,INT,INT.");
690
  printf("%s%s%s%s\n", indent, "-w", indent,
691
          "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
692 693 694
  printf("%s%s%s%s%d\n", indent, "-l", indent,
          "The number of columns per record. Default is 4. Max values is ",
       MAX_NUM_DATATYPE);
695
  printf("%s%s%s%s\n", indent, "-T", indent,
696
          "The number of threads. Default is 10.");
697
  printf("%s%s%s%s\n", indent, "-i", indent,
698
          "The sleep time (ms) between insertion. Default is 0.");
699
  printf("%s%s%s%s\n", indent, "-r", indent,
700
          "The number of records per request. Default is 30000.");
701
  printf("%s%s%s%s\n", indent, "-t", indent,
702
          "The number of tables. Default is 10000.");
703
  printf("%s%s%s%s\n", indent, "-n", indent,
704 705 706
          "The number of records per table. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
  printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
707
  printf("%s%s%s%s\n", indent, "-O", indent,
708
          "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order.");
709
  printf("%s%s%s%s\n", indent, "-R", indent,
710
          "Out of order data's range, ms, default is 1000.");
711
  printf("%s%s%s%s\n", indent, "-g", indent,
712
          "Print debug info.");
713
  printf("%s%s%s\n", indent, "-V, --version\t",
714
          "Print version info.");
715 716
  printf("%s%s%s%s\n", indent, "--help\t", indent,
          "Print command line arguments list info.");
717
/*    printf("%s%s%s%s\n", indent, "-D", indent,
718 719 720 721
          "if elete database if exists. 0: no, 1: yes, default is 1");
          */
}

722 723 724 725 726 727 728 729 730 731 732 733 734 735 736
static bool isStringNumber(char *input)
{
  int len = strlen(input);
  if (0 == len) {
    return false;
  }

  for (int i = 0; i < len; i++) {
    if (!isdigit(input[i]))
      return false;
  }

  return true;
}

737
static void parse_args(int argc, char *argv[], SArguments *arguments) {
738 739 740 741 742

  for (int i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-f") == 0) {
      arguments->metaFile = argv[++i];
    } else if (strcmp(argv[i], "-c") == 0) {
743 744 745 746 747
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-c need a valid path following!\n");
        exit(EXIT_FAILURE);
      }
748
      tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN);
749
    } else if (strcmp(argv[i], "-h") == 0) {
750 751 752 753 754
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-h need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
755 756
      arguments->host = argv[++i];
    } else if (strcmp(argv[i], "-p") == 0) {
757 758 759 760 761 762
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-p need a number following!\n");
        exit(EXIT_FAILURE);
      }
763 764
      arguments->port = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-u") == 0) {
765 766 767 768 769
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-u need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
770 771
      arguments->user = argv[++i];
    } else if (strcmp(argv[i], "-P") == 0) {
772 773 774 775 776
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-P need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
777 778
      arguments->password = argv[++i];
    } else if (strcmp(argv[i], "-o") == 0) {
779 780 781 782 783
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-o need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
784 785
      arguments->output_file = argv[++i];
    } else if (strcmp(argv[i], "-s") == 0) {
786 787 788 789 790
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-s need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
791 792
      arguments->sqlFile = argv[++i];
    } else if (strcmp(argv[i], "-q") == 0) {
793 794
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
795
        printHelp();
796
        errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n");
797 798
        exit(EXIT_FAILURE);
      }
799
      arguments->async_mode = atoi(argv[++i]);
800
    } else if (strcmp(argv[i], "-T") == 0) {
801 802
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
803 804 805 806
        printHelp();
        errorPrint("%s", "\n\t-T need a number following!\n");
        exit(EXIT_FAILURE);
      }
807 808
      arguments->num_of_threads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
809 810
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
811 812 813 814
        printHelp();
        errorPrint("%s", "\n\t-i need a number following!\n");
        exit(EXIT_FAILURE);
      }
815
      arguments->insert_interval = atoi(argv[++i]);
816
    } else if (strcmp(argv[i], "-qt") == 0) {
817 818
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
819
        printHelp();
820
        errorPrint("%s", "\n\t-qt need a number following!\n");
821 822
        exit(EXIT_FAILURE);
      }
823
      arguments->query_times = atoi(argv[++i]);
824
    } else if (strcmp(argv[i], "-B") == 0) {
825 826
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
827 828 829 830
        printHelp();
        errorPrint("%s", "\n\t-B need a number following!\n");
        exit(EXIT_FAILURE);
      }
831
      arguments->interlace_rows = atoi(argv[++i]);
832
    } else if (strcmp(argv[i], "-r") == 0) {
833 834
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
835 836 837 838
        printHelp();
        errorPrint("%s", "\n\t-r need a number following!\n");
        exit(EXIT_FAILURE);
      }
839 840
      arguments->num_of_RPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-t") == 0) {
841 842 843 844 845 846
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-t need a number following!\n");
        exit(EXIT_FAILURE);
      }
847 848
      arguments->num_of_tables = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
849 850 851 852 853 854
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-n need a number following!\n");
        exit(EXIT_FAILURE);
      }
855 856
      arguments->num_of_DPT = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-d") == 0) {
857 858 859 860 861
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-d need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
862 863
      arguments->database = argv[++i];
    } else if (strcmp(argv[i], "-l") == 0) {
864 865 866 867 868 869
      if (argc == i+1) {
        if (!isStringNumber(argv[i+1])) {
            printHelp();
            errorPrint("%s", "\n\t-l need a number following!\n");
            exit(EXIT_FAILURE);
        }
870
      }
871
      arguments->num_of_CPR = atoi(argv[++i]);
872 873 874 875 876 877 878 879 880 881 882

      if (arguments->num_of_CPR > MAX_NUM_DATATYPE) {
          printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE);
          prompt();
          arguments->num_of_CPR = MAX_NUM_DATATYPE;
      }

      for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) {
          arguments->datatype[col] = NULL;
      }

883
    } else if (strcmp(argv[i], "-b") == 0) {
884 885 886 887 888
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-b need valid string following!\n");
        exit(EXIT_FAILURE);
      }
889 890
      ++i;
      if (strstr(argv[i], ",") == NULL) {
891
        // only one col
892 893 894 895 896 897 898
        if (strcasecmp(argv[i], "INT")
                && strcasecmp(argv[i], "FLOAT")
                && strcasecmp(argv[i], "TINYINT")
                && strcasecmp(argv[i], "BOOL")
                && strcasecmp(argv[i], "SMALLINT")
                && strcasecmp(argv[i], "BIGINT")
                && strcasecmp(argv[i], "DOUBLE")
899
                && strcasecmp(argv[i], "BINARY")
900 901
                && strcasecmp(argv[i], "NCHAR")) {
          printHelp();
902
          errorPrint("%s", "-b: Invalid data_type!\n");
903
          exit(EXIT_FAILURE);
904
        }
905
        arguments->datatype[0] = argv[i];
906
      } else {
907
        // more than one col
908
        int index = 0;
909 910
        g_dupstr = strdup(argv[i]);
        char *running = g_dupstr;
911
        char *token = strsep(&running, ",");
912
        while(token != NULL) {
913 914 915 916 917 918 919 920 921 922
          if (strcasecmp(token, "INT")
                  && strcasecmp(token, "FLOAT")
                  && strcasecmp(token, "TINYINT")
                  && strcasecmp(token, "BOOL")
                  && strcasecmp(token, "SMALLINT")
                  && strcasecmp(token, "BIGINT")
                  && strcasecmp(token, "DOUBLE")
                  && strcasecmp(token, "BINARY")
                  && strcasecmp(token, "NCHAR")) {
            printHelp();
923
            free(g_dupstr);
924
            errorPrint("%s", "-b: Invalid data_type!\n");
925
            exit(EXIT_FAILURE);
H
hzcheng 已提交
926
          }
927
          arguments->datatype[index++] = token;
928 929
          token = strsep(&running, ",");
          if (index >= MAX_NUM_DATATYPE) break;
S
TD-1057  
Shengliang Guan 已提交
930
        }
931
        arguments->datatype[index] = NULL;
932
      }
933
    } else if (strcmp(argv[i], "-w") == 0) {
934 935 936 937 938 939
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-w need a number following!\n");
        exit(EXIT_FAILURE);
      }
940 941
      arguments->len_of_binary = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-m") == 0) {
942 943 944 945 946 947
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-m need a number following!\n");
        exit(EXIT_FAILURE);
      }
948
      arguments->tb_prefix = argv[++i];
949 950
    } else if (strcmp(argv[i], "-N") == 0) {
      arguments->use_metric = false;
951
    } else if (strcmp(argv[i], "-x") == 0) {
952
      arguments->insert_only = false;
953
    } else if (strcmp(argv[i], "-y") == 0) {
954
      arguments->answer_yes = true;
955 956
    } else if (strcmp(argv[i], "-g") == 0) {
      arguments->debug_print = true;
957 958
    } else if (strcmp(argv[i], "-gg") == 0) {
      arguments->verbose_print = true;
959 960
    } else if (strcmp(argv[i], "-pp") == 0) {
      arguments->performance_print = true;
961
    } else if (strcmp(argv[i], "-O") == 0) {
962 963 964 965 966 967
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-O need a number following!\n");
        exit(EXIT_FAILURE);
      }
968

969
      arguments->disorderRatio = atoi(argv[++i]);
970

971
      if (arguments->disorderRatio > 50) {
972
        arguments->disorderRatio = 50;
973
      }
974

975
      if (arguments->disorderRatio < 0) {
976
        arguments->disorderRatio = 0;
977
      }
978

979
    } else if (strcmp(argv[i], "-R") == 0) {
980 981 982 983 984 985
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-R need a number following!\n");
        exit(EXIT_FAILURE);
      }
986

987
      arguments->disorderRange = atoi(argv[++i]);
988 989 990
      if (arguments->disorderRange < 0)
        arguments->disorderRange = 1000;

991
    } else if (strcmp(argv[i], "-a") == 0) {
992 993 994 995 996 997
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-a need a number following!\n");
        exit(EXIT_FAILURE);
      }
998 999 1000
      arguments->replica = atoi(argv[++i]);
      if (arguments->replica > 3 || arguments->replica < 1) {
          arguments->replica = 1;
1001
      }
1002 1003
    } else if (strcmp(argv[i], "-D") == 0) {
      arguments->method_of_delete = atoi(argv[++i]);
1004 1005 1006
      if (arguments->method_of_delete > 3) {
        errorPrint("%s", "\n\t-D need a valud (0~3) number following!\n");
        exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1007
      }
1008 1009 1010 1011
    } else if ((strcmp(argv[i], "--version") == 0) ||
        (strcmp(argv[i], "-V") == 0)){
      printVersion();
      exit(0);
1012 1013 1014 1015 1016
    } else if (strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else {
      printHelp();
1017
      errorPrint("%s", "ERROR: wrong options\n");
1018
      exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1019 1020
    }
  }
1021

1022 1023
  if (((arguments->debug_print) && (arguments->metaFile == NULL))
          || arguments->verbose_print) {
1024
    printf("###################################################################\n");
1025
    printf("# meta file:                         %s\n", arguments->metaFile);
1026
    printf("# Server IP:                         %s:%hu\n",
1027 1028 1029 1030 1031
            arguments->host == NULL ? "localhost" : arguments->host,
            arguments->port );
    printf("# User:                              %s\n", arguments->user);
    printf("# Password:                          %s\n", arguments->password);
    printf("# Use metric:                        %s\n", arguments->use_metric ? "true" : "false");
1032 1033 1034 1035 1036 1037 1038 1039 1040
    if (*(arguments->datatype)) {
        printf("# Specified data type:               ");
        for (int i = 0; i < MAX_NUM_DATATYPE; i++)
            if (arguments->datatype[i])
               printf("%s,", arguments->datatype[i]);
            else
                break;
        printf("\n");
    }
1041
    printf("# Insertion interval:                %"PRIu64"\n",
1042
            arguments->insert_interval);
1043
    printf("# Number of records per req:         %"PRIu64"\n",
1044
            arguments->num_of_RPR);
1045
    printf("# Max SQL length:                    %"PRIu64"\n",
1046
            arguments->max_sql_len);
1047
    printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
1048
    printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
1049
    printf("# Number of Tables:                  %"PRId64"\n",
1050
            arguments->num_of_tables);
1051
    printf("# Number of Data per Table:          %"PRId64"\n",
1052
            arguments->num_of_DPT);
1053 1054 1055 1056 1057
    printf("# Database name:                     %s\n", arguments->database);
    printf("# Table prefix:                      %s\n", arguments->tb_prefix);
    if (arguments->disorderRatio) {
      printf("# Data order:                        %d\n", arguments->disorderRatio);
      printf("# Data out of order rate:            %d\n", arguments->disorderRange);
1058

1059 1060 1061 1062
    }
    printf("# Delete method:                     %d\n", arguments->method_of_delete);
    printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
    printf("# Print debug info:                  %d\n", arguments->debug_print);
1063
    printf("# Print verbose info:                %d\n", arguments->verbose_print);
1064
    printf("###################################################################\n");
1065 1066

    prompt();
1067
  }
1068
}
H
hzcheng 已提交
1069

1070 1071 1072 1073
static bool getInfoFromJsonFile(char* file);
//static int generateOneRowDataForStb(SSuperTable* stbInfo);
//static int getDataIntoMemForStb(SSuperTable* stbInfo);
static void init_rand_data();
1074
static void tmfclose(FILE *fp) {
1075 1076 1077 1078
  if (NULL != fp) {
    fclose(fp);
  }
}
1079

1080
static void tmfree(char *buf) {
1081 1082
  if (NULL != buf) {
    free(buf);
H
Hui Li 已提交
1083
  }
1084
}
H
Hui Li 已提交
1085

1086
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
1087 1088 1089
  int i;
  TAOS_RES *res = NULL;
  int32_t   code = -1;
1090

1091 1092 1093 1094
  for (i = 0; i < 5; i++) {
    if (NULL != res) {
      taos_free_result(res);
      res = NULL;
H
hzcheng 已提交
1095
    }
1096

1097 1098 1099 1100
    res = taos_query(taos, command);
    code = taos_errno(res);
    if (0 == code) {
      break;
1101
    }
H
hzcheng 已提交
1102 1103
  }

1104
  verbosePrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
1105
  if (code != 0) {
1106
    if (!quiet) {
1107
      errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res));
1108
    }
1109 1110 1111
    taos_free_result(res);
    //taos_close(taos);
    return -1;
1112
  }
H
hzcheng 已提交
1113

1114 1115 1116 1117
  if (INSERT_TYPE == type) {
    int affectedRows = taos_affected_rows(res);
    taos_free_result(res);
    return affectedRows;
H
hzcheng 已提交
1118
  }
1119

1120
  taos_free_result(res);
1121 1122
  return 0;
}
H
hzcheng 已提交
1123

1124
static void appendResultBufToFile(char *resultBuf, threadInfo *pThreadInfo)
1125
{
1126 1127
  pThreadInfo->fp = fopen(pThreadInfo->filePath, "at");
  if (pThreadInfo->fp == NULL) {
1128 1129
      errorPrint(
              "%s() LN%d, failed to open result file: %s, result will not save to file\n",
1130
              __func__, __LINE__, pThreadInfo->filePath);
1131 1132
      return;
  }
1133

1134 1135 1136
  fprintf(pThreadInfo->fp, "%s", resultBuf);
  tmfclose(pThreadInfo->fp);
  pThreadInfo->fp = NULL;
1137
}
1138

1139
static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) {
1140 1141 1142 1143 1144
  TAOS_ROW    row = NULL;
  int         num_rows = 0;
  int         num_fields = taos_field_count(res);
  TAOS_FIELD *fields     = taos_fetch_fields(res);

1145 1146
  char* databuf = (char*) calloc(1, 100*1024*1024);
  if (databuf == NULL) {
1147 1148
    errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n",
            __func__, __LINE__);
1149 1150
    return ;
  }
H
hzcheng 已提交
1151

1152 1153
  int   totalLen = 0;
  char  temp[16000];
H
hzcheng 已提交
1154

1155
  // fetch the records row by row
1156
  while((row = taos_fetch_row(res))) {
1157
    if ((strlen(pThreadInfo->filePath) > 0)
1158
            && (totalLen >= 100*1024*1024 - 32000)) {
1159
        appendResultBufToFile(databuf, pThreadInfo);
1160 1161
        totalLen = 0;
        memset(databuf, 0, 100*1024*1024);
H
hzcheng 已提交
1162
    }
1163 1164 1165 1166 1167 1168
    num_rows++;
    int len = taos_print_row(temp, row, fields, num_fields);
    len += sprintf(temp + len, "\n");
    //printf("query result:%s\n", temp);
    memcpy(databuf + totalLen, temp, len);
    totalLen += len;
H
hzcheng 已提交
1169 1170
  }

1171
  verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n",
1172 1173 1174
          __func__, __LINE__, databuf, pThreadInfo->filePath);
  if (strlen(pThreadInfo->filePath) > 0) {
    appendResultBufToFile(databuf, pThreadInfo);
1175
  }
1176 1177
  free(databuf);
}
H
hzcheng 已提交
1178

1179
static void selectAndGetResult(
1180
        threadInfo *pThreadInfo, char *command)
1181 1182 1183 1184 1185 1186 1187 1188 1189 1190
{
  if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) {
    TAOS_RES *res = taos_query(pThreadInfo->taos, command);
    if (res == NULL || taos_errno(res) != 0) {
        errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n",
            __func__, __LINE__, command, taos_errstr(res));
        taos_free_result(res);
        return;
    }

1191
    fetchResult(res, pThreadInfo);
1192
    taos_free_result(res);
1193

1194 1195 1196 1197
  } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
      int retCode = postProceSql(
              g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port,
              command,
1198
              pThreadInfo);
1199 1200 1201 1202 1203 1204 1205 1206
      if (0 != retCode) {
        printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID);
      }

  } else {
      errorPrint("%s() LN%d, unknown query mode: %s\n",
        __func__, __LINE__, g_queryInfo.queryMode);
  }
1207
}
H
hzcheng 已提交
1208

1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247
static int32_t rand_bool(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 2;
}

static int32_t rand_tinyint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 128;
}

static int32_t rand_smallint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 32767;
}

static int32_t rand_int(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor];
}

static int64_t rand_bigint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randbigint[cursor];
}

static float rand_float(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
1248
  return randfloat[cursor];
1249 1250
}

1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267
#if 0
static const char charNum[] = "0123456789";

static void nonrand_string(char *, int) __attribute__ ((unused));   // reserve for debugging purpose
static void nonrand_string(char *str, int size)
{
  str[0] = 0;
  if (size > 0) {
    int n;
    for (n = 0; n < size; n++) {
      str[n] = charNum[n % 10];
    }
    str[n] = 0;
  }
}
#endif

1268
static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1269

1270
static void rand_string(char *str, int size) {
1271 1272 1273 1274
  str[0] = 0;
  if (size > 0) {
    //--size;
    int n;
1275
    for (n = 0; n < size; n++) {
1276
      int key = abs(rand_tinyint()) % (int)(sizeof(charset) - 1);
1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
      str[n] = charset[key];
    }
    str[n] = 0;
  }
}

static double rand_double() {
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randdouble[cursor];

}

static void init_rand_data() {
  for (int i = 0; i < MAX_PREPARED_RAND; i++){
1293 1294 1295 1296
    randint[i] = (int)(taosRandom() % 65535);
    randbigint[i] = (int64_t)(taosRandom() % 2147483648);
    randfloat[i] = (float)(taosRandom() / 1000.0);
    randdouble[i] = (double)(taosRandom() / 1000000.0);
1297 1298 1299
  }
}

1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1320
static int printfInsertMeta() {
1321 1322
    SHOW_PARSE_RESULT_START();

1323 1324
  printf("host:                       \033[33m%s:%u\033[0m\n",
          g_Dbs.host, g_Dbs.port);
1325 1326
  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
1327
  printf("configDir:                  \033[33m%s\033[0m\n", configDir);
1328 1329
  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
1330 1331
  printf("thread num of create table: \033[33m%d\033[0m\n",
          g_Dbs.threadCountByCreateTbl);
1332
  printf("top insert interval:        \033[33m%"PRIu64"\033[0m\n",
1333
          g_args.insert_interval);
1334
  printf("number of records per req:  \033[33m%"PRIu64"\033[0m\n",
1335
          g_args.num_of_RPR);
1336
  printf("max sql length:             \033[33m%"PRIu64"\033[0m\n",
1337
          g_args.max_sql_len);
1338 1339

  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
1340

1341 1342
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    printf("database[\033[33m%d\033[0m]:\n", i);
1343 1344
    printf("  database[%d] name:      \033[33m%s\033[0m\n",
            i, g_Dbs.db[i].dbName);
1345
    if (0 == g_Dbs.db[i].drop) {
1346 1347 1348
      printf("  drop:                  \033[33mno\033[0m\n");
    } else {
      printf("  drop:                  \033[33myes\033[0m\n");
1349 1350 1351
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
1352 1353
      printf("  blocks:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.blocks);
1354 1355
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
1356 1357
      printf("  cache:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.cache);
1358 1359
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
1360 1361
      printf("  days:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.days);
1362 1363
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
1364 1365
      printf("  keep:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.keep);
1366 1367
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
1368 1369
      printf("  replica:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.replica);
1370 1371
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
1372 1373
      printf("  update:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.update);
1374 1375
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
1376 1377
      printf("  minRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.minRows);
1378 1379
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
1380 1381
      printf("  maxRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.maxRows);
1382 1383 1384 1385 1386
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
1387 1388
      printf("  walLevel:              \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.walLevel);
1389 1390
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
1391 1392
      printf("  fsync:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.fsync);
1393 1394
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
1395 1396
      printf("  quorum:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.quorum);
1397 1398
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1399 1400
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1401 1402
        printf("  precision:             \033[33m%s\033[0m\n",
            g_Dbs.db[i].dbCfg.precision);
1403
      } else {
1404 1405 1406
        printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
                g_Dbs.db[i].dbCfg.precision);
        return -1;
1407 1408 1409
      }
    }

1410
    printf("  super table count:     \033[33m%"PRIu64"\033[0m\n",
1411
        g_Dbs.db[i].superTblCount);
1412 1413
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      printf("  super table[\033[33m%"PRIu64"\033[0m]:\n", j);
1414

1415 1416
      printf("      stbName:           \033[33m%s\033[0m\n",
          g_Dbs.db[i].superTbls[j].sTblName);
1417 1418 1419 1420 1421 1422 1423 1424

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
      }
1425

1426 1427 1428 1429 1430 1431 1432
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
      }
1433

1434
      printf("      childTblCount:     \033[33m%"PRId64"\033[0m\n",
1435 1436 1437 1438 1439 1440 1441
              g_Dbs.db[i].superTbls[j].childTblCount);
      printf("      childTblPrefix:    \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      printf("      dataSource:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      printf("      insertMode:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].insertMode);
1442
      if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
1443
        printf("      childTblLimit:     \033[33m%"PRId64"\033[0m\n",
1444
                g_Dbs.db[i].superTbls[j].childTblLimit);
1445
      }
1446
      if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) {
1447
        printf("      childTblOffset:    \033[33m%"PRIu64"\033[0m\n",
1448
                g_Dbs.db[i].superTbls[j].childTblOffset);
1449
      }
1450
      printf("      insertRows:        \033[33m%"PRId64"\033[0m\n",
1451
              g_Dbs.db[i].superTbls[j].insertRows);
1452
/*
1453
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1454
        printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
1455
      }else {
1456
        printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
1457
      }
1458
      */
1459
      printf("      interlaceRows:     \033[33m%"PRIu64"\033[0m\n",
1460
              g_Dbs.db[i].superTbls[j].interlaceRows);
1461 1462

      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1463
        printf("      stable insert interval:   \033[33m%"PRIu64"\033[0m\n",
1464 1465 1466
            g_Dbs.db[i].superTbls[j].insertInterval);
      }

1467 1468 1469 1470
      printf("      disorderRange:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      printf("      disorderRatio:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
1471
      printf("      maxSqlLen:         \033[33m%"PRIu64"\033[0m\n",
1472
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1473
      printf("      timeStampStep:     \033[33m%"PRId64"\033[0m\n",
1474 1475 1476 1477 1478 1479 1480 1481 1482
              g_Dbs.db[i].superTbls[j].timeStampStep);
      printf("      startTimestamp:    \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
      printf("      sampleFormat:      \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      printf("      sampleFile:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      printf("      tagsFile:          \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
1483
      printf("      columnCount:       \033[33m%d\033[0m\n",
1484
              g_Dbs.db[i].superTbls[j].columnCount);
1485 1486
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1487 1488 1489 1490
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "nchar", 5))) {
1491
          printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
1492 1493
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1494
        } else {
1495 1496
          printf("column[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType);
1497 1498 1499
        }
      }
      printf("\n");
1500 1501 1502

      printf("      tagCount:            \033[33m%d\033[0m\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1503 1504
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1505 1506 1507 1508
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1509
          printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
1510 1511
                  g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1512
        } else {
1513 1514
          printf("tag[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].tags[k].dataType);
1515
        }
1516 1517 1518 1519 1520
      }
      printf("\n");
    }
    printf("\n");
  }
1521 1522

  SHOW_PARSE_RESULT_END();
1523 1524

  return 0;
1525 1526 1527
}

static void printfInsertMetaToFile(FILE* fp) {
1528 1529

  SHOW_PARSE_RESULT_START_TO_FILE(fp);
1530

1531 1532
  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
  fprintf(fp, "user:                       %s\n", g_Dbs.user);
1533
  fprintf(fp, "configDir:                  %s\n", configDir);
1534 1535 1536
  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
1537 1538
  fprintf(fp, "number of records per req:  %"PRIu64"\n", g_args.num_of_RPR);
  fprintf(fp, "max sql length:             %"PRIu64"\n", g_args.max_sql_len);
1539
  fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
1540

1541 1542
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    fprintf(fp, "database[%d]:\n", i);
1543
    fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
1544
    if (0 == g_Dbs.db[i].drop) {
1545
      fprintf(fp, "  drop:                  no\n");
1546
    }else {
1547
      fprintf(fp, "  drop:                  yes\n");
1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1587 1588
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1589 1590
        fprintf(fp, "  precision:             %s\n",
                g_Dbs.db[i].dbCfg.precision);
1591
      } else {
1592 1593
        fprintf(fp, "  precision error:       %s\n",
                g_Dbs.db[i].dbCfg.precision);
1594 1595 1596
      }
    }

1597 1598
    fprintf(fp, "  super table count:     %"PRIu64"\n",
            g_Dbs.db[i].superTblCount);
1599 1600
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      fprintf(fp, "  super table[%d]:\n", j);
1601

1602 1603
      fprintf(fp, "      stbName:           %s\n",
              g_Dbs.db[i].superTbls[j].sTblName);
1604 1605 1606

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "no");
1607 1608
      } else if (AUTO_CREATE_SUBTBL
              == g_Dbs.db[i].superTbls[j].autoCreateTable) {
1609 1610 1611 1612
        fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
      } else {
        fprintf(fp, "      autoCreateTable:   %s\n",  "error");
      }
1613

1614 1615
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "no");
1616 1617
      } else if (TBL_ALREADY_EXISTS
              == g_Dbs.db[i].superTbls[j].childTblExists) {
1618 1619 1620 1621
        fprintf(fp, "      childTblExists:    %s\n",  "yes");
      } else {
        fprintf(fp, "      childTblExists:    %s\n",  "error");
      }
1622

1623
      fprintf(fp, "      childTblCount:     %"PRId64"\n",
1624 1625 1626 1627 1628 1629 1630
              g_Dbs.db[i].superTbls[j].childTblCount);
      fprintf(fp, "      childTblPrefix:    %s\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      fprintf(fp, "      dataSource:        %s\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      fprintf(fp, "      insertMode:        %s\n",
              g_Dbs.db[i].superTbls[j].insertMode);
1631
      fprintf(fp, "      insertRows:        %"PRId64"\n",
1632
              g_Dbs.db[i].superTbls[j].insertRows);
1633
      fprintf(fp, "      interlace rows:    %"PRIu64"\n",
1634
              g_Dbs.db[i].superTbls[j].interlaceRows);
1635
      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1636
        fprintf(fp, "      stable insert interval:   %"PRIu64"\n",
1637
                g_Dbs.db[i].superTbls[j].insertInterval);
1638
      }
1639
/*
1640
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1641
        fprintf(fp, "      multiThreadWriteOneTbl:  no\n");
1642
      }else {
1643
        fprintf(fp, "      multiThreadWriteOneTbl:  yes\n");
1644
      }
1645
      */
1646
      fprintf(fp, "      interlaceRows:     %"PRIu64"\n",
1647
              g_Dbs.db[i].superTbls[j].interlaceRows);
1648 1649 1650 1651
      fprintf(fp, "      disorderRange:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      fprintf(fp, "      disorderRatio:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
1652
      fprintf(fp, "      maxSqlLen:         %"PRIu64"\n",
1653
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1654

1655 1656 1657 1658
      fprintf(fp, "      timeStampStep:     %"PRId64"\n",
              g_Dbs.db[i].superTbls[j].timeStampStep);
      fprintf(fp, "      startTimestamp:    %s\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
1659 1660 1661 1662 1663 1664
      fprintf(fp, "      sampleFormat:      %s\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      fprintf(fp, "      sampleFile:        %s\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      fprintf(fp, "      tagsFile:          %s\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
1665

1666 1667
      fprintf(fp, "      columnCount:       %d\n        ",
              g_Dbs.db[i].superTbls[j].columnCount);
1668 1669
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1670 1671 1672
        if ((0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "binary", strlen("binary")))
1673 1674
                || (0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
1675 1676 1677 1678
                        "nchar", strlen("nchar")))) {
          fprintf(fp, "column[%d]:%s(%d) ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1679
        } else {
1680 1681
          fprintf(fp, "column[%d]:%s ",
                  k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
1682 1683 1684
        }
      }
      fprintf(fp, "\n");
1685 1686 1687

      fprintf(fp, "      tagCount:            %d\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1688 1689
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1690 1691 1692 1693
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1694 1695
          fprintf(fp, "tag[%d]:%s(%d) ",
                  k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
1696
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1697 1698
        } else {
          fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
1699
        }
1700 1701 1702 1703 1704
      }
      fprintf(fp, "\n");
    }
    fprintf(fp, "\n");
  }
1705

1706
  SHOW_PARSE_RESULT_END_TO_FILE(fp);
1707 1708 1709
}

static void printfQueryMeta() {
1710

1711
  SHOW_PARSE_RESULT_START();
1712

1713 1714
  printf("host:                    \033[33m%s:%u\033[0m\n",
          g_queryInfo.host, g_queryInfo.port);
1715 1716 1717 1718
  printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
  printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

  printf("\n");
1719 1720 1721

  if ((SUBSCRIBE_TEST == g_args.test_mode) || (QUERY_TEST == g_args.test_mode)) {
    printf("specified table query info:                   \n");
1722
    printf("sqlCount:       \033[33m%d\033[0m\n",
1723
      g_queryInfo.specifiedQueryInfo.sqlCount);
1724 1725 1726
    if (g_queryInfo.specifiedQueryInfo.sqlCount > 0) {
      printf("specified tbl query times:\n");
      printf("                \033[33m%"PRIu64"\033[0m\n",
1727
      g_queryInfo.specifiedQueryInfo.queryTimes);
1728 1729 1730
      printf("query interval: \033[33m%"PRIu64" ms\033[0m\n",
        g_queryInfo.specifiedQueryInfo.queryInterval);
      printf("top query times:\033[33m%"PRIu64"\033[0m\n", g_args.query_times);
1731
      printf("concurrent:     \033[33m%d\033[0m\n",
1732 1733
      g_queryInfo.specifiedQueryInfo.concurrent);
      printf("mod:            \033[33m%s\033[0m\n",
1734
        (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync");
1735
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1736
        g_queryInfo.specifiedQueryInfo.subscribeInterval);
1737
      printf("restart:        \033[33m%d\033[0m\n",
1738
        g_queryInfo.specifiedQueryInfo.subscribeRestart);
1739
      printf("keepProgress:   \033[33m%d\033[0m\n",
1740
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
1741

1742 1743
      for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        printf("  sql[%d]: \033[33m%s\033[0m\n",
1744 1745 1746 1747 1748 1749
            i, g_queryInfo.specifiedQueryInfo.sql[i]);
      }
      printf("\n");
    }

    printf("super table query info:\n");
1750
    printf("sqlCount:       \033[33m%d\033[0m\n",
1751 1752 1753 1754 1755 1756 1757
      g_queryInfo.superQueryInfo.sqlCount);

    if (g_queryInfo.superQueryInfo.sqlCount > 0) {
      printf("query interval: \033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryInterval);
      printf("threadCnt:      \033[33m%d\033[0m\n",
        g_queryInfo.superQueryInfo.threadCnt);
1758
      printf("childTblCount:  \033[33m%"PRId64"\033[0m\n",
1759 1760 1761 1762 1763 1764 1765
        g_queryInfo.superQueryInfo.childTblCount);
      printf("stable name:    \033[33m%s\033[0m\n",
        g_queryInfo.superQueryInfo.sTblName);
      printf("stb query times:\033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryTimes);

      printf("mod:            \033[33m%s\033[0m\n",
1766
        (g_queryInfo.superQueryInfo.asyncMode)?"async":"sync");
1767
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1768
        g_queryInfo.superQueryInfo.subscribeInterval);
1769
      printf("restart:        \033[33m%d\033[0m\n",
1770
        g_queryInfo.superQueryInfo.subscribeRestart);
1771
      printf("keepProgress:   \033[33m%d\033[0m\n",
1772 1773
        g_queryInfo.superQueryInfo.subscribeKeepProgress);

1774 1775 1776 1777 1778 1779
      for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
        printf("  sql[%d]: \033[33m%s\033[0m\n",
            i, g_queryInfo.superQueryInfo.sql[i]);
      }
      printf("\n");
    }
1780
  }
1781

1782
  SHOW_PARSE_RESULT_END();
1783 1784
}

1785
static char* formatTimestamp(char* buf, int64_t val, int precision) {
1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816
  time_t tt;
  if (precision == TSDB_TIME_PRECISION_MICRO) {
    tt = (time_t)(val / 1000000);
  } else {
    tt = (time_t)(val / 1000);
  }

/* comment out as it make testcases like select_with_tags.sim fail.
  but in windows, this may cause the call to localtime crash if tt < 0,
  need to find a better solution.
  if (tt < 0) {
    tt = 0;
  }
  */

#ifdef WINDOWS
  if (tt < 0) tt = 0;
#endif

  struct tm* ptm = localtime(&tt);
  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);

  if (precision == TSDB_TIME_PRECISION_MICRO) {
    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
  } else {
    sprintf(buf + pos, ".%03d", (int)(val % 1000));
  }

  return buf;
}

1817 1818 1819
static void xDumpFieldToFile(FILE* fp, const char* val,
        TAOS_FIELD* field, int32_t length, int precision) {

1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854
  if (val == NULL) {
    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
    return;
  }

  char buf[TSDB_MAX_BYTES_PER_ROW];
  switch (field->type) {
    case TSDB_DATA_TYPE_BOOL:
      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
      break;
    case TSDB_DATA_TYPE_TINYINT:
      fprintf(fp, "%d", *((int8_t *)val));
      break;
    case TSDB_DATA_TYPE_SMALLINT:
      fprintf(fp, "%d", *((int16_t *)val));
      break;
    case TSDB_DATA_TYPE_INT:
      fprintf(fp, "%d", *((int32_t *)val));
      break;
    case TSDB_DATA_TYPE_BIGINT:
      fprintf(fp, "%" PRId64, *((int64_t *)val));
      break;
    case TSDB_DATA_TYPE_FLOAT:
      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
      break;
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      memcpy(buf, val, length);
      buf[length] = 0;
      fprintf(fp, "\'%s\'", buf);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
1855
      formatTimestamp(buf, *(int64_t*)val, precision);
1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870
      fprintf(fp, "'%s'", buf);
      break;
    default:
      break;
  }
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  FILE* fp = fopen(fname, "at");
  if (fp == NULL) {
1871
    errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885
    return -1;
  }

  int num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int precision = taos_result_precision(tres);

  for (int col = 0; col < num_fields; col++) {
    if (col > 0) {
      fprintf(fp, ",");
    }
    fprintf(fp, "%s", fields[col].name);
  }
  fputc('\n', fp);
1886

1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906
  int numOfRows = 0;
  do {
    int32_t* length = taos_fetch_lengths(tres);
    for (int i = 0; i < num_fields; i++) {
      if (i > 0) {
        fputc(',', fp);
      }
      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
    }
    fputc('\n', fp);

    numOfRows++;
    row = taos_fetch_row(tres);
  } while( row != NULL);

  fclose(fp);

  return numOfRows;
}

1907 1908
static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
  TAOS_RES * res;
1909 1910
  TAOS_ROW row = NULL;
  int count = 0;
1911 1912

  res = taos_query(taos, "show databases;");
1913
  int32_t code = taos_errno(res);
1914

1915
  if (code != 0) {
1916
    errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
1917 1918 1919 1920 1921
    return -1;
  }

  TAOS_FIELD *fields = taos_fetch_fields(res);

1922
  while((row = taos_fetch_row(res)) != NULL) {
1923
    // sys database name : 'log'
1924 1925 1926 1927
    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log",
                fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) {
      continue;
    }
1928 1929 1930

    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
    if (dbInfos[count] == NULL) {
1931
      errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
1932 1933 1934
      return -1;
    }

1935 1936
    tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
            fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
1937
    formatTimestamp(dbInfos[count]->create_time,
1938 1939
            *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
            TSDB_TIME_PRECISION_MILLI);
1940
    dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
1941
    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
1942 1943
    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
1944
    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
1945

1946
    tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
1947
            fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
1948 1949 1950 1951 1952 1953 1954
    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
1955 1956
    dbInfos[count]->cachelast =
      (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));
1957

1958
    tstrncpy(dbInfos[count]->precision,
1959
            (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
1960
            fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
1961
    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
1962
    tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
1963
            fields[TSDB_SHOW_DB_STATUS_INDEX].bytes);
1964

1965 1966
    count++;
    if (count > MAX_DATABASE_COUNT) {
1967 1968
      errorPrint("%s() LN%d, The database count overflow than %d\n",
         __func__, __LINE__, MAX_DATABASE_COUNT);
1969 1970 1971 1972 1973 1974 1975
      break;
    }
  }

  return count;
}

1976 1977
static void printfDbInfoForQueryToFile(
        char* filename, SDbInfo* dbInfos, int index) {
1978

1979
  if (filename[0] == 0)
1980
      return;
1981 1982 1983

  FILE *fp = fopen(filename, "at");
  if (fp == NULL) {
1984
    errorPrint( "failed to open file: %s\n", filename);
1985
    return;
1986 1987 1988 1989 1990
  }

  fprintf(fp, "================ database[%d] ================\n", index);
  fprintf(fp, "name: %s\n", dbInfos->name);
  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
1991
  fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables);
1992
  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);
1993 1994
  fprintf(fp, "replica: %d\n", dbInfos->replica);
  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
1995 1996
  fprintf(fp, "days: %d\n", dbInfos->days);
  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);
1997 1998 1999 2000 2001 2002 2003
  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
  fprintf(fp, "comp: %d\n", dbInfos->comp);
2004 2005
  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);
  fprintf(fp, "precision: %s\n", dbInfos->precision);
2006
  fprintf(fp, "update: %d\n", dbInfos->update);
2007
  fprintf(fp, "status: %s\n", dbInfos->status);
2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021
  fprintf(fp, "\n");

  fclose(fp);
}

static void printfQuerySystemInfo(TAOS * taos) {
  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
  TAOS_RES* res;

  time_t t;
  struct tm* lt;
  time(&t);
  lt = localtime(&t);
2022 2023 2024 2025
  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
          lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
          lt->tm_sec);

2026 2027
  // show variables
  res = taos_query(taos, "show variables;");
2028
  //fetchResult(res, filename);
2029 2030 2031 2032 2033
  xDumpResultToFile(filename, res);

  // show dnodes
  res = taos_query(taos, "show dnodes;");
  xDumpResultToFile(filename, res);
2034
  //fetchResult(res, filename);
2035

2036 2037 2038 2039
  // show databases
  res = taos_query(taos, "show databases;");
  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
  if (dbInfos == NULL) {
2040
    errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
2041 2042 2043
    return;
  }
  int dbCount = getDbFromServer(taos, dbInfos);
2044 2045 2046 2047
  if (dbCount <= 0) {
      free(dbInfos);
      return;
  }
2048 2049

  for (int i = 0; i < dbCount; i++) {
2050
    // printf database info
2051
    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
2052

2053 2054 2055 2056
    // show db.vgroups
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);
2057

2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068
    // show db.stables
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);

    free(dbInfos[i]);
  }

  free(dbInfos);
}

2069
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
2070
        char* sqlstr, threadInfo *pThreadInfo)
2071
{
2072
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
2073 2074 2075

    char *url = "/rest/sql";

2076
    int bytes, sent, received, req_str_len, resp_len;
2077 2078 2079
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
2080

2081 2082 2083
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;

    request_buf = malloc(req_buf_len);
2084 2085 2086 2087
    if (NULL == request_buf) {
      errorPrint("%s", "ERROR, cannot allocate memory.\n");
      exit(EXIT_FAILURE);
    }
2088

2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};

    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
2100

2101 2102 2103 2104 2105 2106
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);

    char base64_buf[INPUT_BUF_LEN];
2107
#ifdef WINDOWS
2108
    WSADATA wsaData;
2109
    WSAStartup(MAKEWORD(2, 2), &wsaData);
2110 2111 2112 2113
    SOCKET sockfd;
#else
    int sockfd;
#endif
2114
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
2115
    if (sockfd < 0) {
2116
#ifdef WINDOWS
2117
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
2118
#endif
2119
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
2120
        free(request_buf);
2121
        ERROR_EXIT("ERROR opening socket");
2122
    }
2123

2124
    int retConn = connect(sockfd, (struct sockaddr *)pServAddr, sizeof(struct sockaddr));
2125 2126
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
2127
        free(request_buf);
2128
        ERROR_EXIT("ERROR connecting");
2129
    }
2130

2131 2132 2133
    memset(base64_buf, 0, INPUT_BUF_LEN);

    for (int n = 0, m = 0; n < userpass_buf_len;) {
2134
      uint32_t oct_a = n < userpass_buf_len ?
2135
        (unsigned char) userpass_buf[n++]:0;
2136
      uint32_t oct_b = n < userpass_buf_len ?
2137
        (unsigned char) userpass_buf[n++]:0;
2138
      uint32_t oct_c = n < userpass_buf_len ?
2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;

      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }

    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';

2151 2152
    debugPrint("%s() LN%d: auth string base64 encoded: %s\n",
            __func__, __LINE__, base64_buf);
2153 2154
    char *auth = base64_buf;

2155 2156 2157
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
2158 2159 2160 2161 2162
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
2163
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
2164

2165
    req_str_len = strlen(request_buf);
2166 2167
    sent = 0;
    do {
2168 2169 2170
#ifdef WINDOWS
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
#else
2171
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
2172
#endif
2173
        if (bytes < 0)
2174
            ERROR_EXIT("ERROR writing message to socket");
2175 2176 2177
        if (bytes == 0)
            break;
        sent+=bytes;
2178
    } while(sent < req_str_len);
2179

2180 2181
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
2182 2183
    received = 0;
    do {
2184 2185 2186
#ifdef WINDOWS
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
#else
2187
        bytes = read(sockfd, response_buf + received, resp_len - received);
2188
#endif
2189 2190
        if (bytes < 0) {
            free(request_buf);
2191
            ERROR_EXIT("ERROR reading response from socket");
2192
        }
2193 2194
        if (bytes == 0)
            break;
2195
        received += bytes;
2196
    } while(received < resp_len);
2197

2198 2199
    if (received == resp_len) {
        free(request_buf);
2200
        ERROR_EXIT("ERROR storing complete response from socket");
2201
    }
2202

2203
    response_buf[RESP_BUF_LEN - 1] = '\0';
2204 2205
    printf("Response:\n%s\n", response_buf);

2206 2207
    if (strlen(pThreadInfo->filePath) > 0) {
       appendResultBufToFile(response_buf, pThreadInfo);
2208 2209
    }

2210
    free(request_buf);
2211 2212 2213 2214
#ifdef WINDOWS
    closesocket(sockfd);
    WSACleanup();
#else
2215
    close(sockfd);
2216
#endif
2217 2218 2219 2220

    return 0;
}

2221
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
2222 2223
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
2224 2225
    errorPrint("%s() LN%d, calloc failed! size:%d\n",
        __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
2226 2227
    return NULL;
  }
2228

2229
  int    dataLen = 0;
2230 2231 2232
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);

2233 2234 2235
  return dataBuf;
}

2236
static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) {
2237 2238 2239 2240 2241 2242 2243 2244 2245
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
    printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
    return NULL;
  }

  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
  for (int i = 0; i < stbInfo->tagCount; i++) {
2246 2247
    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
2248
      if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
2249 2250
        printf("binary or nchar length overflow, max size:%u\n",
                (uint32_t)TSDB_MAX_BINARY_LEN);
2251 2252 2253
        tmfree(dataBuf);
        return NULL;
      }
2254

2255 2256
      int tagBufLen = stbInfo->tags[i].dataLen + 1;
      char* buf = (char*)calloc(tagBufLen, 1);
2257 2258 2259 2260 2261
      if (NULL == buf) {
        printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
        tmfree(dataBuf);
        return NULL;
      }
2262 2263 2264 2265 2266 2267 2268

      if (tableSeq % 2) {
        tstrncpy(buf, "beijing", tagBufLen);
      } else {
        tstrncpy(buf, "shanghai", tagBufLen);
      }
      //rand_string(buf, stbInfo->tags[i].dataLen);
2269 2270
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "\'%s\', ", buf);
2271
      tmfree(buf);
2272 2273 2274
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "int", strlen("int"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2275
              "%d, ", tableSeq);
2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bigint", strlen("bigint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "float", strlen("float"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_float());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "double", strlen("double"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_double());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "smallint", strlen("smallint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_smallint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "tinyint", strlen("tinyint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_tinyint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bool", strlen("bool"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_bool());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "timestamp", strlen("timestamp"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
2304 2305 2306 2307 2308 2309
    }  else {
      printf("No support data type: %s\n", stbInfo->tags[i].dataType);
      tmfree(dataBuf);
      return NULL;
    }
  }
2310

2311
  dataLen -= 2;
2312
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");
2313 2314 2315
  return dataBuf;
}

2316
static int calcRowLen(SSuperTable*  superTbls) {
2317 2318
  int colIndex;
  int  lenOfOneRow = 0;
2319

2320 2321
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2322

2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
2339
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2340
      lenOfOneRow += 42;
2341
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
2355

2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
2372
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2373 2374 2375 2376 2377 2378 2379 2380
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2381

2382 2383 2384 2385
  return 0;
}


2386 2387
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
2388
        int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) {
2389

2390
  char command[BUFFER_SIZE] = "\0";
2391 2392
  char limitBuf[100] = "\0";

2393
  TAOS_RES * res;
2394 2395 2396
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2397 2398

  if (offset >= 0) {
2399
    snprintf(limitBuf, 100, " limit %"PRId64" offset %"PRIu64"",
2400
            limit, offset);
2401 2402
  }

2403
  //get all child table name use cmd: select tbname from superTblName;
2404 2405
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s",
          dbName, sTblName, limitBuf);
2406

2407
  res = taos_query(taos, command);
2408 2409 2410 2411
  int32_t code = taos_errno(res);
  if (code != 0) {
    taos_free_result(res);
    taos_close(taos);
2412 2413
    errorPrint("%s() LN%d, failed to run command %s\n",
           __func__, __LINE__, command);
2414 2415 2416
    exit(-1);
  }

2417 2418
  int64_t childTblCount = (limit < 0)?10000:limit;
  int64_t count = 0;
2419 2420 2421 2422 2423 2424 2425 2426 2427 2428
  if (childTblName == NULL) {
    childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (NULL ==  childTblName) {
    taos_free_result(res);
        taos_close(taos);
        errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__);
        exit(-1);
    }
  }

2429
  char* pTblName = childTblName;
2430
  while((row = taos_fetch_row(res)) != NULL) {
2431
    int32_t* len = taos_fetch_lengths(res);
2432
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2433 2434 2435
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2436 2437
      char *tmp = realloc(childTblName,
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2438 2439 2440
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2441 2442
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2443 2444
      } else {
        // exit, if allocate more memory failed
2445 2446
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
               __func__, __LINE__, dbName, sTblName);
2447 2448 2449 2450 2451 2452
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2453
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2454
  }
2455

2456 2457 2458 2459 2460 2461 2462
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2463 2464
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
        char* sTblName, char** childTblNameOfSuperTbl,
2465
        int64_t* childTblCountOfSuperTbl) {
2466 2467 2468

    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
2469
            -1, 0);
2470 2471
}

2472 2473
static int getSuperTableFromServer(TAOS * taos, char* dbName,
        SSuperTable*  superTbls) {
2474

2475
  char command[BUFFER_SIZE] = "\0";
2476
  TAOS_RES * res;
2477 2478
  TAOS_ROW row = NULL;
  int count = 0;
2479

2480
  //get schema use cmd: describe superTblName;
2481
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2482
  res = taos_query(taos, command);
2483 2484 2485
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2486
    taos_free_result(res);
2487 2488 2489 2490 2491 2492
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
2493
  while((row = taos_fetch_row(res)) != NULL) {
2494 2495 2496
    if (0 == count) {
      count++;
      continue;
2497
    }
2498 2499

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510
      tstrncpy(superTbls->tags[tagIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->tags[tagIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2511
      tagIndex++;
2512
    } else {
2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523
      tstrncpy(superTbls->columns[columnIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->columns[columnIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2535
/*
2536
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
2537
    //get all child table name use cmd: select tbname from superTblName;
2538 2539 2540
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2541
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2542 2543
      return -1;
    }
2544
    getAllChildNameOfSuperTable(taos, dbName,
2545 2546 2547
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2548
  }
2549
  */
2550 2551 2552
  return 0;
}

2553 2554
static int createSuperTable(
        TAOS * taos, char* dbName,
2555
        SSuperTable*  superTbl) {
2556

2557
  char command[BUFFER_SIZE] = "\0";
2558

2559 2560 2561 2562 2563
  char cols[STRING_LEN] = "\0";
  int colIndex;
  int len = 0;

  int  lenOfOneRow = 0;
2564 2565 2566 2567 2568 2569 2570 2571 2572

  if (superTbl->columnCount == 0) {
    errorPrint("%s() LN%d, super table column count is %d\n",
            __func__, __LINE__, superTbl->columnCount);
    return -1;
  }

  for (colIndex = 0; colIndex < superTbl->columnCount; colIndex++) {
    char* dataType = superTbl->columns[colIndex].dataType;
2573

2574
    if (strcasecmp(dataType, "BINARY") == 0) {
2575 2576
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "BINARY",
2577 2578
          superTbl->columns[colIndex].dataLen);
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2579
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
2580 2581
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "NCHAR",
2582 2583
          superTbl->columns[colIndex].dataLen);
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
      lenOfOneRow += 22;
2602
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2603 2604
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
      lenOfOneRow += 42;
2605
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2606 2607 2608 2609
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
      lenOfOneRow += 21;
    } else {
      taos_close(taos);
2610 2611
      errorPrint("%s() LN%d, config error data type : %s\n",
         __func__, __LINE__, dataType);
2612 2613 2614 2615
      exit(-1);
    }
  }

2616 2617
  superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp
  //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbl[j].sTblName, g_Dbs.db[i].superTbl[j].columnCount, lenOfOneRow);
2618 2619

  // save for creating child table
2620 2621 2622 2623
  superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1);
  if (NULL == superTbl->colsOfCreateChildTable) {
    errorPrint("%s() LN%d, Failed when calloc, size:%d",
           __func__, __LINE__, len+1);
2624 2625 2626 2627
    taos_close(taos);
    exit(-1);
  }

2628
  snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
2629 2630
  verbosePrint("%s() LN%d: %s\n",
      __func__, __LINE__, superTbl->colsOfCreateChildTable);
2631

2632 2633 2634 2635 2636
  if (superTbl->tagCount == 0) {
    errorPrint("%s() LN%d, super table tag count is %d\n",
            __func__, __LINE__, superTbl->tagCount);
    return -1;
  }
2637

2638 2639 2640
  char tags[STRING_LEN] = "\0";
  int tagIndex;
  len = 0;
2641

2642 2643 2644 2645
  int lenOfTagOfOneRow = 0;
  len += snprintf(tags + len, STRING_LEN - len, "(");
  for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) {
    char* dataType = superTbl->tags[tagIndex].dataType;
2646

2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684
    if (strcasecmp(dataType, "BINARY") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
              "BINARY", superTbl->tags[tagIndex].dataLen);
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
              "NCHAR", superTbl->tags[tagIndex].dataLen);
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "INT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "BIGINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "SMALLINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "TINYINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "BOOL");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "FLOAT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "DOUBLE");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42;
    } else {
      taos_close(taos);
2685 2686
      errorPrint("%s() LN%d, config error tag type : %s\n",
         __func__, __LINE__, dataType);
2687
      exit(-1);
2688 2689
    }
  }
2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704

  len -= 2;
  len += snprintf(tags + len, STRING_LEN - len, ")");

  superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow;

  snprintf(command, BUFFER_SIZE,
          "create table if not exists %s.%s (ts timestamp%s) tags %s",
          dbName, superTbl->sTblName, cols, tags);
  if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
      errorPrint( "create supertable %s failed!\n\n",
              superTbl->sTblName);
      return -1;
  }
  debugPrint("create supertable %s success!\n\n", superTbl->sTblName);
2705 2706 2707
  return 0;
}

2708
static int createDatabasesAndStables() {
2709 2710 2711 2712
  TAOS * taos = NULL;
  int    ret = 0;
  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
  if (taos == NULL) {
2713
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2714
    return -1;
2715 2716 2717
  }
  char command[BUFFER_SIZE] = "\0";

2718
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2719 2720
    if (g_Dbs.db[i].drop) {
      sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
2721
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2722 2723 2724 2725
        taos_close(taos);
        return -1;
      }

2726
      int dataLen = 0;
2727
      dataLen += snprintf(command + dataLen,
2728
          BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2729

2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798
      if (g_Dbs.db[i].dbCfg.blocks > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
      }
      if (g_Dbs.db[i].dbCfg.cache > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
      }
      if (g_Dbs.db[i].dbCfg.days > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
      }
      if (g_Dbs.db[i].dbCfg.keep > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
      }
      if (g_Dbs.db[i].dbCfg.quorum > 1) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
      }
      if (g_Dbs.db[i].dbCfg.replica > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
      }
      if (g_Dbs.db[i].dbCfg.update > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
      }
      //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
      //  dataLen += snprintf(command + dataLen,
      //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
      //}
      if (g_Dbs.db[i].dbCfg.minRows > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
      }
      if (g_Dbs.db[i].dbCfg.maxRows > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
      }
      if (g_Dbs.db[i].dbCfg.comp > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
      }
      if (g_Dbs.db[i].dbCfg.walLevel > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
      }
      if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
      }
      if (g_Dbs.db[i].dbCfg.fsync > 0) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
                " fsync %d", g_Dbs.db[i].dbCfg.fsync);
      }
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                      "us", strlen("us")))) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
                " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
      }

      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
        taos_close(taos);
        errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
        return -1;
      }
      printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
2799 2800
    }

2801
    debugPrint("%s() LN%d supertbl count:%"PRIu64"\n",
2802
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
2803 2804 2805

    int validStbCount = 0;

2806
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
2807 2808 2809 2810 2811
      sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
              g_Dbs.db[i].superTbls[j].sTblName);
      ret = queryDbExec(taos, command, NO_INSERT_TYPE, true);

      if ((ret != 0) || (g_Dbs.db[i].drop)) {
2812
        ret = createSuperTable(taos, g_Dbs.db[i].dbName,
2813
                &g_Dbs.db[i].superTbls[j]);
2814

2815
        if (0 != ret) {
2816 2817
          errorPrint("create super table %d failed!\n\n", j);
          continue;
2818 2819 2820
        }
      }

2821
      ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
2822
                &g_Dbs.db[i].superTbls[j]);
2823 2824 2825
      if (0 != ret) {
        errorPrint("\nget super table %s.%s info failed!\n\n",
                g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName);
2826
        continue;
2827
      }
2828 2829

      validStbCount ++;
2830
    }
2831 2832

    g_Dbs.db[i].superTblCount = validStbCount;
2833 2834 2835 2836 2837 2838
  }

  taos_close(taos);
  return 0;
}

2839 2840
static void* createTable(void *sarg)
{
2841 2842
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
2843 2844 2845

  int64_t  lastPrintTime = taosGetTimestampMs();

2846
  int buff_len;
2847
  buff_len = BUFFER_SIZE / 8;
2848

2849
  char *buffer = calloc(buff_len, 1);
2850
  if (buffer == NULL) {
2851
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
2852 2853
    exit(-1);
  }
2854 2855 2856

  int len = 0;
  int batchNum = 0;
2857

2858
  verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n",
2859
          __func__, __LINE__,
2860
          pThreadInfo->start_table_from, pThreadInfo->end_table_to);
2861

2862
  for (uint64_t i = pThreadInfo->start_table_from;
2863
          i <= pThreadInfo->end_table_to; i++) {
2864
    if (0 == g_Dbs.use_metric) {
2865
      snprintf(buffer, buff_len,
2866
              "create table if not exists %s.%s%"PRIu64" %s;",
2867
              pThreadInfo->db_name,
2868
              g_args.tb_prefix, i,
2869
              pThreadInfo->cols);
2870
    } else {
2871 2872 2873
      if (superTblInfo == NULL) {
        errorPrint("%s() LN%d, use metric, but super table info is NULL\n",
                  __func__, __LINE__);
2874
        free(buffer);
2875 2876 2877 2878 2879 2880 2881 2882 2883 2884
        exit(-1);
      } else {
        if (0 == len) {
          batchNum = 0;
          memset(buffer, 0, buff_len);
          len += snprintf(buffer + len,
                  buff_len - len, "create table ");
        }
        char* tagsValBuf = NULL;
        if (0 == superTblInfo->tagSource) {
2885
          tagsValBuf = generateTagVaulesForStb(superTblInfo, i);
2886 2887 2888 2889 2890 2891 2892 2893 2894 2895
        } else {
          tagsValBuf = getTagValueFromTagSample(
                  superTblInfo,
                  i % superTblInfo->tagSampleCount);
        }
        if (NULL == tagsValBuf) {
          free(buffer);
          return NULL;
        }
        len += snprintf(buffer + len,
2896
                buff_len - len,
2897
                "if not exists %s.%s%"PRIu64" using %s.%s tags %s ",
2898 2899
                pThreadInfo->db_name, superTblInfo->childTblPrefix,
                i, pThreadInfo->db_name,
2900 2901 2902 2903
                superTblInfo->sTblName, tagsValBuf);
        free(tagsValBuf);
        batchNum++;
        if ((batchNum < superTblInfo->batchCreateTableNum)
2904
                && ((buff_len - len)
2905 2906 2907
                    >= (superTblInfo->lenOfTagOfOneRow + 256))) {
          continue;
        }
2908 2909 2910 2911
      }
    }

    len = 0;
2912
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)){
2913
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2914
      free(buffer);
2915 2916 2917 2918 2919
      return NULL;
    }

    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
2920
      printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n",
2921
              pThreadInfo->threadID, pThreadInfo->start_table_from, i);
2922 2923 2924
      lastPrintTime = currentPrintTime;
    }
  }
2925

2926
  if (0 != len) {
2927
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)) {
2928
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2929
    }
2930
  }
2931

2932 2933 2934 2935
  free(buffer);
  return NULL;
}

2936
static int startMultiThreadCreateChildTable(
2937
        char* cols, int threads, uint64_t startFrom, int64_t ntables,
2938
        char* db_name, SSuperTable* superTblInfo) {
2939

2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  threadInfo *infos = malloc(threads * sizeof(threadInfo));

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

2952
  int64_t a = ntables / threads;
2953 2954 2955 2956 2957
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

2958
  int64_t b = 0;
2959
  b = ntables % threads;
2960

2961
  for (int64_t i = 0; i < threads; i++) {
2962 2963 2964 2965
    threadInfo *pThreadInfo = infos + i;
    pThreadInfo->threadID = i;
    tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE);
    pThreadInfo->superTblInfo = superTblInfo;
2966
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
2967
    pThreadInfo->taos = taos_connect(
2968 2969 2970 2971 2972
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
2973
    if (pThreadInfo->taos == NULL) {
2974 2975
      errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n",
         __func__, __LINE__, taos_errstr(NULL));
2976
      free(pids);
2977
      free(infos);
2978 2979
      return -1;
    }
2980

2981 2982 2983 2984 2985 2986 2987 2988
    pThreadInfo->start_table_from = startFrom;
    pThreadInfo->ntables = i<b?a+1:a;
    pThreadInfo->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
    startFrom = pThreadInfo->end_table_to + 1;
    pThreadInfo->use_metric = true;
    pThreadInfo->cols = cols;
    pThreadInfo->minDelay = UINT64_MAX;
    pthread_create(pids + i, NULL, createTable, pThreadInfo);
2989
  }
2990

2991 2992 2993 2994 2995
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
2996 2997
    threadInfo *pThreadInfo = infos + i;
    taos_close(pThreadInfo->taos);
2998 2999 3000
  }

  free(pids);
3001
  free(infos);
3002 3003

  return 0;
3004 3005
}

3006 3007 3008 3009
static void createChildTables() {
    char tblColsBuf[MAX_SQL_SIZE];
    int len;

3010
  for (int i = 0; i < g_Dbs.dbCount; i++) {
3011 3012 3013 3014 3015 3016 3017 3018
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
          // with super table
        for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
          if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
                || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
            continue;
          }
3019

3020 3021
          verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
                  g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
3022
          uint64_t startFrom = 0;
3023 3024
          g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;

3025
          verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n",
3026 3027 3028 3029 3030 3031 3032 3033
                  __func__, __LINE__, g_totalChildTables, startFrom);
          startMultiThreadCreateChildTable(
                g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
                g_Dbs.threadCountByCreateTbl,
                startFrom,
                g_Dbs.db[i].superTbls[j].childTblCount,
                g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
        }
3034 3035
      }
    } else {
3036 3037
      // normal table
      len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
3038
      for (int j = 0; j < g_args.num_of_CPR; j++) {
3039 3040 3041 3042
          if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
                  || (strncasecmp(g_args.datatype[j],
                      "NCHAR", strlen("NCHAR")) == 0)) {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
3043
                      ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary);
3044 3045 3046 3047 3048 3049
          } else {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                      ", COL%d %s", j, g_args.datatype[j]);
          }
          len = strlen(tblColsBuf);
      }
3050

3051
      snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");
3052

3053
      verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n",
3054 3055 3056 3057 3058 3059 3060 3061 3062
              __func__, __LINE__,
              g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
      startMultiThreadCreateChildTable(
            tblColsBuf,
            g_Dbs.threadCountByCreateTbl,
            0,
            g_args.num_of_tables,
            g_Dbs.db[i].dbName,
            NULL);
3063
    }
3064 3065 3066 3067 3068 3069
  }
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3070
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
3071 3072 3073
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
3074

3075 3076
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
3077 3078
    printf("Failed to open tags file: %s, reason:%s\n",
            superTblInfo->tagsFile, strerror(errno));
3079 3080 3081 3082 3083 3084 3085
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
3086

3087 3088 3089 3090 3091 3092 3093 3094 3095
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

3096
  while((readLen = tgetline(&line, &n, fp)) != -1) {
3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
3109 3110
      char *tmp = realloc(tagDataBuf,
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
3111 3112 3113
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
3114 3115
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

int readSampleFromJsonFileToMem(SSuperTable  * superTblInfo) {
  // TODO
  return 0;
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3143 3144
static int readSampleFromCsvFileToMem(
        SSuperTable* superTblInfo) {
3145 3146 3147 3148
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
3149

3150 3151
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
3152
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
3153 3154 3155
              superTblInfo->sampleFile, strerror(errno));
      return -1;
  }
3156

3157
  assert(superTblInfo->sampleDataBuf);
3158 3159
  memset(superTblInfo->sampleDataBuf, 0,
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
3160
  while(1) {
3161
    readLen = tgetline(&line, &n, fp);
3162 3163
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
3164
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
3165
                superTblInfo->sampleFile, strerror(errno));
3166
        fclose(fp);
3167 3168 3169 3170
        return -1;
      }
      continue;
    }
3171

3172 3173 3174 3175 3176 3177 3178 3179 3180
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
3181
      printf("sample row len[%d] overflow define schema len[%"PRIu64"], so discard this row\n",
3182
              (int32_t)readLen, superTblInfo->lenOfOneRow);
3183 3184 3185
      continue;
    }

3186 3187
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
          line, readLen);
3188 3189 3190 3191 3192 3193 3194
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

3195
  fclose(fp);
3196 3197 3198 3199
  tmfree(line);
  return 0;
}

3200 3201
static bool getColumnAndTagTypeFromInsertJsonFile(
        cJSON* stbInfo, SSuperTable* superTbls) {
3202
  bool  ret = false;
3203

3204
  // columns
3205 3206
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
3207
    printf("ERROR: failed to read json, columns not found\n");
3208 3209 3210 3211 3212 3213
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
3214

3215
  int columnSize = cJSON_GetArraySize(columns);
3216
  if ((columnSize + 1/* ts */) > MAX_COLUMN_COUNT) {
3217 3218
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
            __func__, __LINE__, MAX_COLUMN_COUNT);
3219 3220 3221 3222 3223 3224
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
3225

3226
  //superTbls->columnCount = columnSize;
3227 3228 3229 3230 3231 3232 3233
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
3234
      count = countObj->valueint;
3235
    } else if (countObj && countObj->type != cJSON_Number) {
3236 3237
      errorPrint("%s() LN%d, failed to read json, column count not found\n",
          __func__, __LINE__);
3238 3239 3240 3241 3242
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3243
    // column info
3244 3245
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
3246 3247 3248 3249
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d: failed to read json, column type not found\n",
          __func__, __LINE__);
3250 3251
      goto PARSE_OVER;
    }
3252 3253
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3254

3255 3256
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3257
      columnCase.dataLen = dataLen->valueint;
3258
    } else if (dataLen && dataLen->type != cJSON_Number) {
3259 3260
      debugPrint("%s() LN%d: failed to read json, column len not found\n",
          __func__, __LINE__);
3261 3262 3263 3264
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
3265

3266
    for (int n = 0; n < count; ++n) {
3267 3268 3269
      tstrncpy(superTbls->columns[index].dataType,
              columnCase.dataType, MAX_TB_NAME_SIZE);
      superTbls->columns[index].dataLen = columnCase.dataLen;
3270 3271
      index++;
    }
3272
  }
3273

3274 3275
  if ((index + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n",
3276 3277 3278 3279
            __func__, __LINE__, MAX_COLUMN_COUNT);
    goto PARSE_OVER;
  }

3280
  superTbls->columnCount = index;
3281

3282 3283
  count = 1;
  index = 0;
3284
  // tags
3285 3286
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
3287 3288
    errorPrint("%s() LN%d, failed to read json, tags not found\n",
        __func__, __LINE__);
3289 3290
    goto PARSE_OVER;
  }
3291

3292 3293
  int tagSize = cJSON_GetArraySize(tags);
  if (tagSize > MAX_TAG_COUNT) {
3294 3295
    errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n",
        __func__, __LINE__, MAX_TAG_COUNT);
3296 3297
    goto PARSE_OVER;
  }
3298 3299

  //superTbls->tagCount = tagSize;
3300 3301 3302
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
3303

3304 3305 3306
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
3307
      count = countObj->valueint;
3308
    } else if (countObj && countObj->type != cJSON_Number) {
3309
      printf("ERROR: failed to read json, column count not found\n");
3310 3311 3312 3313 3314
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3315
    // column info
3316 3317
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
3318 3319 3320 3321
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d, failed to read json, tag type not found\n",
          __func__, __LINE__);
3322 3323
      goto PARSE_OVER;
    }
3324
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3325

3326 3327
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3328
      columnCase.dataLen = dataLen->valueint;
3329
    } else if (dataLen && dataLen->type != cJSON_Number) {
3330 3331
      errorPrint("%s() LN%d, failed to read json, column len not found\n",
          __func__, __LINE__);
3332 3333 3334
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
3335 3336
    }

3337
    for (int n = 0; n < count; ++n) {
3338 3339
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType,
          MAX_TB_NAME_SIZE);
3340
      superTbls->tags[index].dataLen = columnCase.dataLen;
3341 3342
      index++;
    }
3343
  }
3344 3345

  if (index > MAX_TAG_COUNT) {
3346
    errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n",
3347 3348 3349 3350
        __func__, __LINE__, MAX_TAG_COUNT);
    goto PARSE_OVER;
  }

3351 3352
  superTbls->tagCount = index;

3353 3354 3355
  if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n",
        __func__, __LINE__, MAX_COLUMN_COUNT);
3356 3357
    goto PARSE_OVER;
  }
3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368
  ret = true;

PARSE_OVER:
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3369
    tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3370 3371 3372 3373
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3374
    tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE);
3375
  } else if (!host) {
3376
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3377
  } else {
3378
    printf("ERROR: failed to read json, host not found\n");
3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_Dbs.port = port->valueint;
  } else if (!port) {
    g_Dbs.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3391
    tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE);
3392
  } else if (!user) {
3393
    tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE);
3394 3395 3396 3397
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3398
    tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE);
3399
  } else if (!password) {
3400
    tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE);
3401 3402 3403 3404
  }

  cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
  if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
3405
    tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
3406
  } else if (!resultfile) {
3407
    tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
3408 3409 3410 3411 3412 3413 3414 3415
  }

  cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
  if (threads && threads->type == cJSON_Number) {
    g_Dbs.threadCount = threads->valueint;
  } else if (!threads) {
    g_Dbs.threadCount = 1;
  } else {
3416
    printf("ERROR: failed to read json, threads not found\n");
3417
    goto PARSE_OVER;
3418 3419
  }

3420 3421 3422 3423
  cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
  if (threads2 && threads2->type == cJSON_Number) {
    g_Dbs.threadCountByCreateTbl = threads2->valueint;
  } else if (!threads2) {
3424
    g_Dbs.threadCountByCreateTbl = 1;
3425
  } else {
3426 3427
    errorPrint("%s() LN%d, failed to read json, threads2 not found\n",
            __func__, __LINE__);
3428
    goto PARSE_OVER;
3429
  }
3430

3431 3432
  cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
  if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
3433 3434 3435 3436 3437
    if (gInsertInterval->valueint <0) {
      errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n",
            __func__, __LINE__);
      goto PARSE_OVER;
    }
3438 3439 3440 3441
    g_args.insert_interval = gInsertInterval->valueint;
  } else if (!gInsertInterval) {
    g_args.insert_interval = 0;
  } else {
3442 3443
    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
        __func__, __LINE__);
3444 3445
    goto PARSE_OVER;
  }
3446

3447 3448
  cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows");
  if (interlaceRows && interlaceRows->type == cJSON_Number) {
3449 3450 3451 3452 3453 3454
    if (interlaceRows->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;

    }
3455 3456
    g_args.interlace_rows = interlaceRows->valueint;
  } else if (!interlaceRows) {
3457
    g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3458
  } else {
3459 3460
    errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
        __func__, __LINE__);
3461
    goto PARSE_OVER;
3462
  }
3463

3464 3465
  cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
  if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
3466 3467 3468 3469 3470
    if (maxSqlLen->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
    }
3471 3472
    g_args.max_sql_len = maxSqlLen->valueint;
  } else if (!maxSqlLen) {
3473
    g_args.max_sql_len = (1024*1024);
3474
  } else {
3475 3476
    errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
        __func__, __LINE__);
3477 3478 3479 3480 3481
    goto PARSE_OVER;
  }

  cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
  if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
3482
    if (numRecPerReq->valueint <= 0) {
3483 3484 3485
      errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
3486
    } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) {
3487 3488 3489 3490 3491
      printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n",
              numRecPerReq->valueint, MAX_RECORDS_PER_REQ);
      printf("        number of records per request value will be set to %d\n\n",
              MAX_RECORDS_PER_REQ);
      prompt();
3492
      numRecPerReq->valueint = MAX_RECORDS_PER_REQ;
3493
    }
3494 3495
    g_args.num_of_RPR = numRecPerReq->valueint;
  } else if (!numRecPerReq) {
3496
    g_args.num_of_RPR = MAX_RECORDS_PER_REQ;
3497
  } else {
3498 3499
    errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n",
        __func__, __LINE__);
3500 3501 3502
    goto PARSE_OVER;
  }

3503
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3504
  if (answerPrompt
3505 3506
          && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3507 3508 3509 3510 3511 3512 3513 3514
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
    } else {
      g_args.answer_yes = false;
    }
  } else if (!answerPrompt) {
3515
    g_args.answer_yes = true;   // default is no, mean answer_yes.
3516
  } else {
3517
    errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n");
3518
    goto PARSE_OVER;
3519
  }
3520

3521 3522 3523 3524 3525 3526 3527 3528 3529 3530
  // rows per table need be less than insert batch
  if (g_args.interlace_rows > g_args.num_of_RPR) {
      printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
              g_args.interlace_rows, g_args.num_of_RPR);
      printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
              g_args.num_of_RPR);
      prompt();
      g_args.interlace_rows = g_args.num_of_RPR;
  }

3531 3532
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (!dbs || dbs->type != cJSON_Array) {
3533
    printf("ERROR: failed to read json, databases not found\n");
3534 3535 3536 3537 3538
    goto PARSE_OVER;
  }

  int dbSize = cJSON_GetArraySize(dbs);
  if (dbSize > MAX_DB_COUNT) {
3539
    errorPrint(
3540 3541
            "ERROR: failed to read json, databases size overflow, max database is %d\n",
            MAX_DB_COUNT);
3542 3543 3544 3545 3546 3547 3548 3549
    goto PARSE_OVER;
  }

  g_Dbs.dbCount = dbSize;
  for (int i = 0; i < dbSize; ++i) {
    cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
    if (dbinfos == NULL) continue;

3550
    // dbinfo
3551 3552
    cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
    if (!dbinfo || dbinfo->type != cJSON_Object) {
3553
      printf("ERROR: failed to read json, dbinfo not found\n");
3554 3555
      goto PARSE_OVER;
    }
3556

3557 3558
    cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
    if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
3559
      printf("ERROR: failed to read json, db name not found\n");
3560 3561
      goto PARSE_OVER;
    }
3562
    tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE);
3563 3564 3565

    cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
    if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
3566 3567
      if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) {
        g_Dbs.db[i].drop = true;
3568
      } else {
3569
        g_Dbs.db[i].drop = false;
3570
      }
3571
    } else if (!drop) {
3572
      g_Dbs.db[i].drop = g_args.drop_database;
3573
    } else {
3574 3575
      errorPrint("%s() LN%d, failed to read json, drop input mistake\n",
              __func__, __LINE__);
3576 3577 3578 3579
      goto PARSE_OVER;
    }

    cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
3580 3581 3582 3583
    if (precision && precision->type == cJSON_String
            && precision->valuestring != NULL) {
      tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
              MAX_DB_NAME_SIZE);
3584
    } else if (!precision) {
3585
      //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
3586 3587
      memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE);
    } else {
3588
      printf("ERROR: failed to read json, precision not found\n");
3589 3590 3591 3592 3593 3594 3595 3596 3597
      goto PARSE_OVER;
    }

    cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
    if (update && update->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.update = update->valueint;
    } else if (!update) {
      g_Dbs.db[i].dbCfg.update = -1;
    } else {
3598
      printf("ERROR: failed to read json, update not found\n");
3599 3600 3601 3602 3603 3604 3605 3606 3607
      goto PARSE_OVER;
    }

    cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
    if (replica && replica->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.replica = replica->valueint;
    } else if (!replica) {
      g_Dbs.db[i].dbCfg.replica = -1;
    } else {
3608
      printf("ERROR: failed to read json, replica not found\n");
3609 3610 3611 3612 3613 3614 3615 3616 3617
      goto PARSE_OVER;
    }

    cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
    if (keep && keep->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.keep = keep->valueint;
    } else if (!keep) {
      g_Dbs.db[i].dbCfg.keep = -1;
    } else {
3618
     printf("ERROR: failed to read json, keep not found\n");
3619 3620
     goto PARSE_OVER;
    }
3621

3622 3623 3624 3625 3626 3627
    cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
    if (days && days->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.days = days->valueint;
    } else if (!days) {
      g_Dbs.db[i].dbCfg.days = -1;
    } else {
3628
     printf("ERROR: failed to read json, days not found\n");
3629 3630
     goto PARSE_OVER;
    }
3631

3632 3633 3634 3635 3636 3637
    cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
    if (cache && cache->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cache = cache->valueint;
    } else if (!cache) {
      g_Dbs.db[i].dbCfg.cache = -1;
    } else {
3638
     printf("ERROR: failed to read json, cache not found\n");
3639 3640
     goto PARSE_OVER;
    }
3641

3642 3643 3644 3645 3646 3647
    cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
    if (blocks && blocks->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
    } else if (!blocks) {
      g_Dbs.db[i].dbCfg.blocks = -1;
    } else {
3648
     printf("ERROR: failed to read json, block not found\n");
3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665
     goto PARSE_OVER;
    }

    //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
    //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
    //} else if (!maxtablesPerVnode) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
    //} else {
    // printf("failed to read json, maxtablesPerVnode not found");
    // goto PARSE_OVER;
    //}

    cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
    if (minRows && minRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
    } else if (!minRows) {
3666
      g_Dbs.db[i].dbCfg.minRows = 0;    // 0 means default
3667
    } else {
3668
     printf("ERROR: failed to read json, minRows not found\n");
3669 3670 3671 3672 3673 3674 3675
     goto PARSE_OVER;
    }

    cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
    if (maxRows && maxRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
    } else if (!maxRows) {
3676
      g_Dbs.db[i].dbCfg.maxRows = 0;    // 0 means default
3677
    } else {
3678
     printf("ERROR: failed to read json, maxRows not found\n");
3679 3680 3681 3682 3683 3684 3685 3686 3687
     goto PARSE_OVER;
    }

    cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
    if (comp && comp->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.comp = comp->valueint;
    } else if (!comp) {
      g_Dbs.db[i].dbCfg.comp = -1;
    } else {
3688
     printf("ERROR: failed to read json, comp not found\n");
3689 3690 3691 3692 3693 3694 3695 3696 3697
     goto PARSE_OVER;
    }

    cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
    if (walLevel && walLevel->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
    } else if (!walLevel) {
      g_Dbs.db[i].dbCfg.walLevel = -1;
    } else {
3698
     printf("ERROR: failed to read json, walLevel not found\n");
3699 3700 3701
     goto PARSE_OVER;
    }

3702 3703 3704 3705 3706 3707
    cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
    if (cacheLast && cacheLast->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
    } else if (!cacheLast) {
      g_Dbs.db[i].dbCfg.cacheLast = -1;
    } else {
3708
     printf("ERROR: failed to read json, cacheLast not found\n");
3709 3710 3711
     goto PARSE_OVER;
    }

3712 3713 3714 3715
    cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
    if (quorum && quorum->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
    } else if (!quorum) {
3716
      g_Dbs.db[i].dbCfg.quorum = 1;
3717
    } else {
3718
     printf("failed to read json, quorum input mistake");
3719 3720 3721 3722 3723 3724 3725 3726 3727
     goto PARSE_OVER;
    }

    cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
    if (fsync && fsync->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
    } else if (!fsync) {
      g_Dbs.db[i].dbCfg.fsync = -1;
    } else {
3728 3729 3730
      errorPrint("%s() LN%d, failed to read json, fsync input mistake\n",
              __func__, __LINE__);
      goto PARSE_OVER;
3731
    }
3732

3733
    // super_talbes
3734 3735
    cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
    if (!stables || stables->type != cJSON_Array) {
3736 3737
      errorPrint("%s() LN%d, failed to read json, super_tables not found\n",
              __func__, __LINE__);
3738
      goto PARSE_OVER;
3739 3740
    }

3741 3742
    int stbSize = cJSON_GetArraySize(stables);
    if (stbSize > MAX_SUPER_TABLE_COUNT) {
3743
      errorPrint(
3744 3745
              "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n",
              __func__, __LINE__, MAX_SUPER_TABLE_COUNT);
3746 3747 3748 3749 3750 3751 3752
      goto PARSE_OVER;
    }

    g_Dbs.db[i].superTblCount = stbSize;
    for (int j = 0; j < stbSize; ++j) {
      cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
      if (stbInfo == NULL) continue;
3753 3754

      // dbinfo
3755 3756
      cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
      if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) {
3757 3758
        errorPrint("%s() LN%d, failed to read json, stb name not found\n",
                __func__, __LINE__);
3759 3760
        goto PARSE_OVER;
      }
3761
      tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE);
3762

3763 3764
      cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
      if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
3765
        printf("ERROR: failed to read json, childtable_prefix not found\n");
3766 3767
        goto PARSE_OVER;
      }
3768
      tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE);
3769 3770

      cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null
3771 3772 3773
      if (autoCreateTbl
              && autoCreateTbl->type == cJSON_String
              && autoCreateTbl->valuestring != NULL) {
3774 3775 3776 3777 3778 3779 3780 3781 3782 3783
        if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
        } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        } else {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        }
      } else if (!autoCreateTbl) {
        g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      } else {
3784
        printf("ERROR: failed to read json, auto_create_table not found\n");
3785 3786
        goto PARSE_OVER;
      }
3787

3788 3789 3790 3791
      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
      } else if (!batchCreateTbl) {
3792
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
3793
      } else {
3794
        printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
3795
        goto PARSE_OVER;
3796
      }
3797 3798

      cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
3799 3800 3801
      if (childTblExists
              && childTblExists->type == cJSON_String
              && childTblExists->valuestring != NULL) {
3802 3803
        if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3))
            && (g_Dbs.db[i].drop == false)) {
3804
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
3805 3806
        } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2)
              || (g_Dbs.db[i].drop == true))) {
3807 3808 3809 3810 3811 3812 3813
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        } else {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        }
      } else if (!childTblExists) {
        g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
      } else {
3814 3815
        errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n",
                __func__, __LINE__);
3816 3817
        goto PARSE_OVER;
      }
3818

3819 3820
      cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
      if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
3821
        errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n",
3822
                __func__, __LINE__);
3823 3824 3825 3826 3827
        goto PARSE_OVER;
      }
      g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

      cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
3828 3829 3830 3831
      if (dataSource && dataSource->type == cJSON_String
              && dataSource->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
                dataSource->valuestring, MAX_DB_NAME_SIZE);
3832
      } else if (!dataSource) {
3833
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE);
3834
      } else {
3835 3836
        errorPrint("%s() LN%d, failed to read json, data_source not found\n",
            __func__, __LINE__);
3837 3838 3839
        goto PARSE_OVER;
      }

3840
      cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest
3841 3842 3843 3844
      if (insertMode && insertMode->type == cJSON_String
              && insertMode->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode,
                insertMode->valuestring, MAX_DB_NAME_SIZE);
3845
      } else if (!insertMode) {
3846
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE);
3847
      } else {
3848
        printf("ERROR: failed to read json, insert_mode not found\n");
3849 3850 3851
        goto PARSE_OVER;
      }

3852
      cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
3853 3854
      if ((childTbl_limit) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3855 3856 3857 3858 3859 3860
        if (childTbl_limit->type != cJSON_Number) {
            printf("ERROR: failed to read json, childtable_limit\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
      } else {
3861
        g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid.
3862 3863 3864
      }

      cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
3865 3866
      if ((childTbl_offset) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3867 3868 3869 3870 3871 3872 3873 3874 3875
        if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) {
            printf("ERROR: failed to read json, childtable_offset\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblOffset = 0;
      }

3876 3877
      cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
      if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
3878 3879
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                ts->valuestring, MAX_DB_NAME_SIZE);
3880
      } else if (!ts) {
3881 3882
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                "now", MAX_DB_NAME_SIZE);
3883
      } else {
3884
        printf("ERROR: failed to read json, start_timestamp not found\n");
3885 3886
        goto PARSE_OVER;
      }
3887

3888 3889 3890 3891
      cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
      if (timestampStep && timestampStep->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
      } else if (!timestampStep) {
3892
        g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
3893
      } else {
3894
        printf("ERROR: failed to read json, timestamp_step not found\n");
3895 3896 3897 3898
        goto PARSE_OVER;
      }

      cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
3899 3900
      if (sampleFormat && sampleFormat->type
              == cJSON_String && sampleFormat->valuestring != NULL) {
3901 3902
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
                sampleFormat->valuestring, MAX_DB_NAME_SIZE);
3903
      } else if (!sampleFormat) {
3904
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE);
3905
      } else {
3906
        printf("ERROR: failed to read json, sample_format not found\n");
3907
        goto PARSE_OVER;
3908
      }
3909

3910
      cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
3911 3912
      if (sampleFile && sampleFile->type == cJSON_String
          && sampleFile->valuestring != NULL) {
3913 3914
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
                sampleFile->valuestring, MAX_FILE_NAME_LEN);
3915 3916 3917
      } else if (!sampleFile) {
        memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
      } else {
3918
        printf("ERROR: failed to read json, sample_file not found\n");
3919
        goto PARSE_OVER;
3920
      }
3921

3922 3923
      cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
      if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) {
3924 3925
        tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
                tagsFile->valuestring, MAX_FILE_NAME_LEN);
3926 3927 3928 3929 3930 3931 3932 3933 3934
        if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
          g_Dbs.db[i].superTbls[j].tagSource = 0;
        } else {
          g_Dbs.db[i].superTbls[j].tagSource = 1;
        }
      } else if (!tagsFile) {
        memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
        g_Dbs.db[i].superTbls[j].tagSource = 0;
      } else {
3935
        printf("ERROR: failed to read json, tags_file not found\n");
3936 3937
        goto PARSE_OVER;
      }
3938

3939 3940 3941 3942 3943
      cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
      if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
        int32_t len = maxSqlLen->valueint;
        if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
          len = TSDB_MAX_ALLOWED_SQL_LEN;
3944 3945
        } else if (len < 5) {
          len = 5;
3946
        }
3947 3948
        g_Dbs.db[i].superTbls[j].maxSqlLen = len;
      } else if (!maxSqlLen) {
3949
        g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len;
3950
      } else {
3951 3952
        errorPrint("%s() LN%d, failed to read json, maxSqlLen input mistake\n",
            __func__, __LINE__);
3953
        goto PARSE_OVER;
3954
      }
3955
/*
3956 3957 3958 3959 3960
      cJSON *multiThreadWriteOneTbl =
          cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
      if (multiThreadWriteOneTbl
              && multiThreadWriteOneTbl->type == cJSON_String
              && multiThreadWriteOneTbl->valuestring != NULL) {
3961 3962 3963 3964
        if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
        } else {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
3965
        }
3966 3967 3968
      } else if (!multiThreadWriteOneTbl) {
        g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
      } else {
3969
        printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
3970 3971
        goto PARSE_OVER;
      }
3972
*/
3973 3974
      cJSON* interlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows");
      if (interlaceRows && interlaceRows->type == cJSON_Number) {
3975 3976 3977 3978 3979
        if (interlaceRows->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n",
            __func__, __LINE__);
          goto PARSE_OVER;
        }
3980
        g_Dbs.db[i].superTbls[j].interlaceRows = interlaceRows->valueint;
3981 3982
        // rows per table need be less than insert batch
        if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) {
3983
          printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
3984
                  i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR);
3985
          printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
3986
                  g_args.num_of_RPR);
3987
          prompt();
3988 3989
          g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR;
        }
3990 3991
      } else if (!interlaceRows) {
        g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3992
      } else {
3993 3994 3995
        errorPrint(
                "%s() LN%d, failed to read json, interlace rows input mistake\n",
                __func__, __LINE__);
3996
        goto PARSE_OVER;
3997
      }
3998 3999 4000

      cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
      if (disorderRatio && disorderRatio->type == cJSON_Number) {
4001 4002 4003 4004 4005 4006
        if (disorderRatio->valueint > 50)
          disorderRatio->valueint = 50;

        if (disorderRatio->valueint < 0)
          disorderRatio->valueint = 0;

4007 4008 4009 4010
        g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
      } else if (!disorderRatio) {
        g_Dbs.db[i].superTbls[j].disorderRatio = 0;
      } else {
4011
        printf("ERROR: failed to read json, disorderRatio not found\n");
4012
        goto PARSE_OVER;
4013
      }
4014 4015 4016 4017 4018 4019 4020

      cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
      if (disorderRange && disorderRange->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
      } else if (!disorderRange) {
        g_Dbs.db[i].superTbls[j].disorderRange = 1000;
      } else {
4021
        printf("ERROR: failed to read json, disorderRange not found\n");
4022 4023
        goto PARSE_OVER;
      }
4024

4025 4026
      cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
      if (insertRows && insertRows->type == cJSON_Number) {
4027 4028 4029 4030 4031
        if (insertRows->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4032 4033 4034 4035
        g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
      } else if (!insertRows) {
        g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
      } else {
4036 4037
        errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
4038 4039 4040 4041 4042 4043
        goto PARSE_OVER;
      }

      cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
      if (insertInterval && insertInterval->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
4044 4045 4046 4047 4048
        if (insertInterval->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4049
      } else if (!insertInterval) {
4050
        verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n",
4051 4052 4053
                __func__, __LINE__, g_args.insert_interval);
        g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
      } else {
4054 4055
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
4056 4057 4058
        goto PARSE_OVER;
      }

4059 4060
      int retVal = getColumnAndTagTypeFromInsertJsonFile(
              stbInfo, &g_Dbs.db[i].superTbls[j]);
4061 4062
      if (false == retVal) {
        goto PARSE_OVER;
4063 4064
      }
    }
4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077
  }

  ret = true;

PARSE_OVER:
  return ret;
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
4078
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
4079 4080 4081 4082
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
4083
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE);
4084
  } else if (!host) {
4085
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
4086
  } else {
4087
    printf("ERROR: failed to read json, host not found\n");
4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
4100
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE);
4101
  } else if (!user) {
4102
    tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ;
4103 4104 4105 4106
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
4107
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE);
4108
  } else if (!password) {
4109
    tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);;
4110
  }
H
hzcheng 已提交
4111

4112
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
4113 4114
  if (answerPrompt && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
4115 4116 4117 4118
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
4119
    } else {
4120
      g_args.answer_yes = false;
H
hzcheng 已提交
4121
    }
4122 4123 4124
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
4125
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
4126
    goto PARSE_OVER;
4127
  }
4128

4129 4130
  cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
  if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
4131
    if (gQueryTimes->valueint <= 0) {
4132 4133
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4134 4135
      goto PARSE_OVER;
    }
4136 4137 4138 4139
    g_args.query_times = gQueryTimes->valueint;
  } else if (!gQueryTimes) {
    g_args.query_times = 1;
  } else {
4140 4141
    errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4142 4143 4144
    goto PARSE_OVER;
  }

4145 4146
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
4147
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE);
4148
  } else if (!dbs) {
4149
    printf("ERROR: failed to read json, databases not found\n");
4150
    goto PARSE_OVER;
H
hzcheng 已提交
4151
  }
4152 4153 4154

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
4155
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
4156
  } else if (!queryMode) {
4157
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
4158
  } else {
4159
    printf("ERROR: failed to read json, query_mode not found\n");
4160
    goto PARSE_OVER;
H
hzcheng 已提交
4161
  }
4162

4163
  // specified_table_query
4164 4165
  cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!specifiedQuery) {
4166
    g_queryInfo.specifiedQueryInfo.concurrent = 1;
4167 4168
    g_queryInfo.specifiedQueryInfo.sqlCount = 0;
  } else if (specifiedQuery->type != cJSON_Object) {
4169
    printf("ERROR: failed to read json, super_table_query not found\n");
4170
    goto PARSE_OVER;
4171
  } else {
4172 4173 4174 4175 4176
    cJSON* queryInterval = cJSON_GetObjectItem(specifiedQuery, "query_interval");
    if (queryInterval && queryInterval->type == cJSON_Number) {
      g_queryInfo.specifiedQueryInfo.queryInterval = queryInterval->valueint;
    } else if (!queryInterval) {
      g_queryInfo.specifiedQueryInfo.queryInterval = 0;
4177 4178
    }

4179 4180
    cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery,
        "query_times");
4181
    if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
4182
      if (specifiedQueryTimes->valueint <= 0) {
4183 4184
        errorPrint(
                "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4185
          __func__, __LINE__, specifiedQueryTimes->valueint);
4186 4187 4188
        goto PARSE_OVER;

      }
4189 4190 4191 4192
      g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint;
    } else if (!specifiedQueryTimes) {
      g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times;
    } else {
4193 4194
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
          __func__, __LINE__);
4195
      goto PARSE_OVER;
4196
    }
4197

4198
    cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent");
4199
    if (concurrent && concurrent->type == cJSON_Number) {
4200
      if (concurrent->valueint <= 0) {
4201
        errorPrint(
4202
                "%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n",
4203 4204
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount,
4205 4206 4207
              g_queryInfo.specifiedQueryInfo.concurrent);
        goto PARSE_OVER;
      }
4208
      g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint;
4209
    } else if (!concurrent) {
4210
      g_queryInfo.specifiedQueryInfo.concurrent = 1;
4211
    }
4212

4213 4214 4215 4216 4217 4218 4219
    cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode");
    if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String
        && specifiedAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE;
4220
      } else {
4221
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4222
            __func__, __LINE__);
4223 4224 4225
        goto PARSE_OVER;
      }
    } else {
4226
      g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
4227
    }
4228

4229
    cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval");
4230
    if (interval && interval->type == cJSON_Number) {
4231
      g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint;
4232
    } else if (!interval) {
4233 4234
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4235
      g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000;
4236
    }
4237

4238
    cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart");
4239
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
4240
      if (0 == strcmp("yes", restart->valuestring)) {
4241
        g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4242
      } else if (0 == strcmp("no", restart->valuestring)) {
4243
        g_queryInfo.specifiedQueryInfo.subscribeRestart = false;
4244
      } else {
4245
        printf("ERROR: failed to read json, subscribe restart error\n");
4246 4247 4248
        goto PARSE_OVER;
      }
    } else {
4249
      g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4250
    }
4251

4252
    cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress");
4253 4254 4255
    if (keepProgress
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
4256
      if (0 == strcmp("yes", keepProgress->valuestring)) {
4257
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1;
4258
      } else if (0 == strcmp("no", keepProgress->valuestring)) {
4259
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4260
      } else {
4261
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4262 4263 4264
        goto PARSE_OVER;
      }
    } else {
4265
      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4266 4267
    }

4268
    // sqls
4269 4270
    cJSON* specifiedSqls = cJSON_GetObjectItem(specifiedQuery, "sqls");
    if (!specifiedSqls) {
4271
      g_queryInfo.specifiedQueryInfo.sqlCount = 0;
4272
    } else if (specifiedSqls->type != cJSON_Array) {
4273 4274
      errorPrint("%s() LN%d, failed to read json, super sqls not found\n",
          __func__, __LINE__);
4275
      goto PARSE_OVER;
4276
    } else {
4277 4278 4279 4280 4281 4282 4283 4284
      int superSqlSize = cJSON_GetArraySize(specifiedSqls);
      if (superSqlSize * g_queryInfo.specifiedQueryInfo.concurrent
              > MAX_QUERY_SQL_COUNT) {
        errorPrint("%s() LN%d, failed to read json, query sql(%d) * concurrent(%d) overflow, max is %d\n",
           __func__, __LINE__,
           superSqlSize,
           g_queryInfo.specifiedQueryInfo.concurrent,
           MAX_QUERY_SQL_COUNT);
4285 4286
        goto PARSE_OVER;
      }
4287

4288
      g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize;
4289
      for (int j = 0; j < superSqlSize; ++j) {
4290
        cJSON* sql = cJSON_GetArrayItem(specifiedSqls, j);
4291
        if (sql == NULL) continue;
4292

4293 4294
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
4295
          printf("ERROR: failed to read json, sql not found\n");
4296 4297
          goto PARSE_OVER;
        }
4298 4299 4300
        tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j],
                sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);

4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311
        cJSON* endAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "endAfterConsume");
        if (endAfterConsume
                && endAfterConsume->type == cJSON_Number) {
            g_queryInfo.specifiedQueryInfo.endAfterConsume[j]
                = endAfterConsume->valueint;
        } else if (!endAfterConsume) {
            // default value is -1, which mean infinite loop
            g_queryInfo.specifiedQueryInfo.endAfterConsume[j] = -1;
        }

4312 4313 4314 4315 4316 4317 4318
        cJSON* resubAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume");
        if (resubAfterConsume
                && resubAfterConsume->type == cJSON_Number) {
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j]
                = resubAfterConsume->valueint;
        } else if (!resubAfterConsume) {
4319 4320
            // default value is -1, which mean do not resub
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = -1;
4321
        }
H
hzcheng 已提交
4322

4323
        cJSON *result = cJSON_GetObjectItem(sql, "result");
4324 4325 4326 4327
        if ((NULL != result) && (result->type == cJSON_String)
                && (result->valuestring != NULL)) {
          tstrncpy(g_queryInfo.specifiedQueryInfo.result[j],
                  result->valuestring, MAX_FILE_NAME_LEN);
4328
        } else if (NULL == result) {
4329 4330
          memset(g_queryInfo.specifiedQueryInfo.result[j],
                  0, MAX_FILE_NAME_LEN);
4331
        } else {
4332
          printf("ERROR: failed to read json, super query result file not found\n");
4333
          goto PARSE_OVER;
4334
        }
4335 4336 4337
      }
    }
  }
4338

4339
  // super_table_query
4340 4341
  cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!superQuery) {
4342
    g_queryInfo.superQueryInfo.threadCnt = 1;
4343 4344
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
4345
    printf("ERROR: failed to read json, sub_table_query not found\n");
4346 4347
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
4348
  } else {
4349
    cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval");
4350
    if (subrate && subrate->type == cJSON_Number) {
4351
      g_queryInfo.superQueryInfo.queryInterval = subrate->valueint;
4352
    } else if (!subrate) {
4353
      g_queryInfo.superQueryInfo.queryInterval = 0;
4354 4355 4356 4357
    }

    cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
    if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
4358 4359 4360
      if (superQueryTimes->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
          __func__, __LINE__, superQueryTimes->valueint);
4361 4362
        goto PARSE_OVER;
      }
4363 4364 4365 4366
      g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
    } else if (!superQueryTimes) {
      g_queryInfo.superQueryInfo.queryTimes = g_args.query_times;
    } else {
4367 4368
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
          __func__, __LINE__);
4369
      goto PARSE_OVER;
4370
    }
4371

4372
    cJSON* threads = cJSON_GetObjectItem(superQuery, "threads");
4373
    if (threads && threads->type == cJSON_Number) {
4374 4375 4376 4377 4378 4379
      if (threads->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, threads input mistake\n",
          __func__, __LINE__);
        goto PARSE_OVER;

      }
4380
      g_queryInfo.superQueryInfo.threadCnt = threads->valueint;
4381
    } else if (!threads) {
4382
      g_queryInfo.superQueryInfo.threadCnt = 1;
4383
    }
4384

4385
    //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count");
4386
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
4387
    //  g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint;
4388
    //} else if (!subTblCnt) {
4389
    //  g_queryInfo.superQueryInfo.childTblCount = 0;
4390
    //}
4391

4392
    cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname");
4393 4394 4395 4396
    if (stblname && stblname->type == cJSON_String
        && stblname->valuestring != NULL) {
      tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring,
          MAX_TB_NAME_SIZE);
4397
    } else {
4398 4399
      errorPrint("%s() LN%d, failed to read json, super table name input error\n",
          __func__, __LINE__);
4400 4401
      goto PARSE_OVER;
    }
4402

4403 4404 4405 4406 4407 4408 4409
    cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode");
    if (superAsyncMode && superAsyncMode->type == cJSON_String
        && superAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE;
4410
      } else {
4411
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4412
            __func__, __LINE__);
4413 4414 4415
        goto PARSE_OVER;
      }
    } else {
4416
      g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
4417
    }
4418

4419 4420 4421 4422 4423 4424 4425 4426 4427
    cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval");
    if (superInterval && superInterval->type == cJSON_Number) {
      if (superInterval->valueint < 0) {
        errorPrint("%s() LN%d, failed to read json, interval input mistake\n",
            __func__, __LINE__);
        goto PARSE_OVER;
      }
      g_queryInfo.superQueryInfo.subscribeInterval = superInterval->valueint;
    } else if (!superInterval) {
4428 4429
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4430
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
4431
    }
4432

4433
    cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart");
4434 4435
    if (subrestart && subrestart->type == cJSON_String
        && subrestart->valuestring != NULL) {
4436
      if (0 == strcmp("yes", subrestart->valuestring)) {
4437
        g_queryInfo.superQueryInfo.subscribeRestart = true;
4438
      } else if (0 == strcmp("no", subrestart->valuestring)) {
4439
        g_queryInfo.superQueryInfo.subscribeRestart = false;
4440
      } else {
4441
        printf("ERROR: failed to read json, subscribe restart error\n");
4442 4443 4444
        goto PARSE_OVER;
      }
    } else {
4445
      g_queryInfo.superQueryInfo.subscribeRestart = true;
4446
    }
4447

4448 4449 4450 4451 4452
    cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
    if (superkeepProgress &&
            superkeepProgress->type == cJSON_String
            && superkeepProgress->valuestring != NULL) {
      if (0 == strcmp("yes", superkeepProgress->valuestring)) {
4453
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
4454
      } else if (0 == strcmp("no", superkeepProgress->valuestring)) {
4455
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4456
      } else {
4457
        printf("ERROR: failed to read json, subscribe super table keepProgress error\n");
4458 4459 4460
        goto PARSE_OVER;
      }
    } else {
4461
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4462
    }
4463

4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474
    cJSON* superEndAfterConsume =
            cJSON_GetObjectItem(superQuery, "endAfterConsume");
    if (superEndAfterConsume
            && superEndAfterConsume->type == cJSON_Number) {
        g_queryInfo.superQueryInfo.endAfterConsume =
            superEndAfterConsume->valueint;
    } else if (!superEndAfterConsume) {
        // default value is -1, which mean do not resub
        g_queryInfo.superQueryInfo.endAfterConsume = -1;
    }

4475
    cJSON* superResubAfterConsume =
4476
            cJSON_GetObjectItem(superQuery, "endAfterConsume");
4477 4478
    if (superResubAfterConsume
            && superResubAfterConsume->type == cJSON_Number) {
4479
        g_queryInfo.superQueryInfo.endAfterConsume =
4480 4481
            superResubAfterConsume->valueint;
    } else if (!superResubAfterConsume) {
4482 4483
        // default value is -1, which mean do not resub
        g_queryInfo.superQueryInfo.endAfterConsume = -1;
4484 4485
    }

4486 4487 4488
    // supert table sqls
    cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls");
    if (!superSqls) {
4489
      g_queryInfo.superQueryInfo.sqlCount = 0;
4490
    } else if (superSqls->type != cJSON_Array) {
4491 4492
      errorPrint("%s() LN%d: failed to read json, super sqls not found\n",
          __func__, __LINE__);
4493
      goto PARSE_OVER;
4494
    } else {
4495
      int superSqlSize = cJSON_GetArraySize(superSqls);
4496
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4497 4498
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4499 4500
        goto PARSE_OVER;
      }
4501

4502
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
4503
      for (int j = 0; j < superSqlSize; ++j) {
4504
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
4505
        if (sql == NULL) continue;
4506

4507
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
4508 4509 4510 4511
        if (!sqlStr || sqlStr->type != cJSON_String
            || sqlStr->valuestring == NULL) {
          errorPrint("%s() LN%d, failed to read json, sql not found\n",
              __func__, __LINE__);
4512 4513
          goto PARSE_OVER;
        }
4514 4515
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring,
            MAX_QUERY_SQL_LENGTH);
4516 4517

        cJSON *result = cJSON_GetObjectItem(sql, "result");
4518 4519 4520 4521
        if (result != NULL && result->type == cJSON_String
            && result->valuestring != NULL){
          tstrncpy(g_queryInfo.superQueryInfo.result[j],
              result->valuestring, MAX_FILE_NAME_LEN);
4522
        } else if (NULL == result) {
4523
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4524
        }  else {
4525 4526
          errorPrint("%s() LN%d, failed to read json, sub query result file not found\n",
              __func__, __LINE__);
4527
          goto PARSE_OVER;
4528
        }
4529 4530
      }
    }
H
hzcheng 已提交
4531 4532
  }

4533
  ret = true;
H
hzcheng 已提交
4534

4535 4536 4537
PARSE_OVER:
  return ret;
}
H
hzcheng 已提交
4538

4539
static bool getInfoFromJsonFile(char* file) {
4540
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
4541

4542 4543 4544 4545 4546
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
4547

4548
  bool  ret = false;
4549
  int   maxLen = 6400000;
4550 4551 4552 4553 4554 4555 4556
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
4557
  }
H
Hui Li 已提交
4558

4559 4560 4561
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
4562
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
4563 4564
    goto PARSE_OVER;
  }
H
Hui Li 已提交
4565

4566 4567 4568
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
4569
      g_args.test_mode = INSERT_TEST;
4570
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
4571
      g_args.test_mode = QUERY_TEST;
4572
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
4573
      g_args.test_mode = SUBSCRIBE_TEST;
4574
    } else {
4575
      printf("ERROR: failed to read json, filetype not support\n");
4576 4577 4578
      goto PARSE_OVER;
    }
  } else if (!filetype) {
4579
    g_args.test_mode = INSERT_TEST;
4580
  } else {
4581
    printf("ERROR: failed to read json, filetype not found\n");
4582 4583
    goto PARSE_OVER;
  }
H
hzcheng 已提交
4584

4585
  if (INSERT_TEST == g_args.test_mode) {
4586
    ret = getMetaFromInsertJsonFile(root);
4587 4588
  } else if ((QUERY_TEST == g_args.test_mode)
          || (SUBSCRIBE_TEST == g_args.test_mode)) {
4589 4590
    ret = getMetaFromQueryJsonFile(root);
  } else {
4591 4592
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n",
            __func__, __LINE__);
4593
    goto PARSE_OVER;
4594
  }
4595 4596 4597 4598

PARSE_OVER:
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
4599
  fclose(fp);
4600 4601
  return ret;
}
H
hzcheng 已提交
4602

4603
static void prepareSampleData() {
4604
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4605 4606 4607
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
        (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]);
S
Shuaiqiang Chang 已提交
4608
      }
4609 4610 4611 4612
    }
  }
}

4613
static void postFreeResource() {
4614
  tmfclose(g_fpOfInsertResult);
4615
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4616
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
4617 4618 4619
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
4620
      }
4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4633
    }
4634 4635
  }
}
S
Shuaiqiang Chang 已提交
4636

4637 4638 4639
static int getRowDataFromSample(
        char* dataBuf, int64_t maxLen, int64_t timestamp,
      SSuperTable* superTblInfo, int64_t* sampleUsePos) {
4640
  if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
4641
/*    int ret = readSampleFromCsvFileToMem(superTblInfo);
4642
    if (0 != ret) {
4643
      tmfree(superTblInfo->sampleDataBuf);
4644
      superTblInfo->sampleDataBuf = NULL;
4645 4646
      return -1;
    }
4647
*/
4648 4649
    *sampleUsePos = 0;
  }
S
Shuaiqiang Chang 已提交
4650

4651
  int    dataLen = 0;
4652

4653 4654 4655 4656
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "(%" PRId64 ", ", timestamp);
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
4657
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
S
Shuaiqiang Chang 已提交
4658

4659
  (*sampleUsePos)++;
4660

4661 4662
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4663

4664 4665
static int64_t generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stbInfo) {
  int64_t   dataLen = 0;
4666
  char  *pstr = recBuf;
4667
  int64_t maxLen = MAX_DATA_SIZE;
4668

4669
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "(%" PRId64 ",", timestamp);
4670

4671
  for (int i = 0; i < stbInfo->columnCount; i++) {
4672 4673
    if ((0 == strncasecmp(stbInfo->columns[i].dataType, "BINARY", strlen("BINARY")))
            || (0 == strncasecmp(stbInfo->columns[i].dataType, "NCHAR", strlen("NCHAR")))) {
4674
      if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
4675
        errorPrint( "binary or nchar length overflow, max size:%u\n",
4676
                (uint32_t)TSDB_MAX_BINARY_LEN);
4677
        return -1;
4678
      }
4679

4680 4681
      char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
      if (NULL == buf) {
4682
        errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
4683
        return -1;
4684 4685
      }
      rand_string(buf, stbInfo->columns[i].dataLen);
4686
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf);
4687
      tmfree(buf);
4688
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4689
                "INT", 3)) {
4690
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4691
              "%d,", rand_int());
4692
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4693
                "BIGINT", 6)) {
4694
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4695
              "%"PRId64",", rand_bigint());
4696
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4697
                "FLOAT", 5)) {
4698
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4699
              "%f,", rand_float());
4700
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4701
                "DOUBLE", 6)) {
4702
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4703
              "%f,", rand_double());
4704
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4705
                "SMALLINT", 8)) {
4706
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4707
          "%d,", rand_smallint());
4708
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4709
          "TINYINT", strlen("TINYINT"))) {
4710
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4711
          "%d,", rand_tinyint());
4712
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4713
          "BOOL", strlen("BOOL"))) {
4714
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4715
          "%d,", rand_bool());
4716
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4717
          "TIMESTAMP", strlen("TIMESTAMP"))) {
4718
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4719
          "%"PRId64",", rand_bigint());
4720
    }  else {
4721
      errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType);
4722
      return -1;
4723
    }
S
Shuaiqiang Chang 已提交
4724
  }
4725

4726
  dataLen -= 1;
4727
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")");
4728

4729 4730 4731
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

  return strlen(recBuf);
4732
}
S
Shuaiqiang Chang 已提交
4733

4734
static int64_t generateData(char *recBuf, char **data_type,
4735
        int num_of_cols, int64_t timestamp, int lenOfBinary) {
4736 4737
  memset(recBuf, 0, MAX_DATA_SIZE);
  char *pstr = recBuf;
4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751
  pstr += sprintf(pstr, "(%" PRId64, timestamp);
  int c = 0;

  for (; c < MAX_NUM_DATATYPE; c++) {
    if (data_type[c] == NULL) {
      break;
    }
  }

  if (0 == c) {
    perror("data type error!");
    exit(-1);
  }

4752
  for (int i = 0; i < c; i++) {
4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763
    if (strcasecmp(data_type[i % c], "TINYINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_tinyint() );
    } else if (strcasecmp(data_type[i % c], "SMALLINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_smallint());
    } else if (strcasecmp(data_type[i % c], "INT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_int());
    } else if (strcasecmp(data_type[i % c], "BIGINT") == 0) {
      pstr += sprintf(pstr, ",%" PRId64, rand_bigint());
    } else if (strcasecmp(data_type[i % c], "FLOAT") == 0) {
      pstr += sprintf(pstr, ",%10.4f", rand_float());
    } else if (strcasecmp(data_type[i % c], "DOUBLE") == 0) {
4764
      double t = rand_double();
4765 4766
      pstr += sprintf(pstr, ",%20.8f", t);
    } else if (strcasecmp(data_type[i % c], "BOOL") == 0) {
4767
      bool b = rand_bool() & 1;
4768 4769
      pstr += sprintf(pstr, ",%s", b ? "true" : "false");
    } else if (strcasecmp(data_type[i % c], "BINARY") == 0) {
4770 4771
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4772
      pstr += sprintf(pstr, ",\"%s\"", s);
4773
      free(s);
4774
    } else if (strcasecmp(data_type[i % c], "NCHAR") == 0) {
4775 4776
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4777
      pstr += sprintf(pstr, ",\"%s\"", s);
4778 4779 4780
      free(s);
    }

4781
    if (strlen(recBuf) > MAX_DATA_SIZE) {
4782 4783 4784 4785 4786 4787 4788
      perror("column length too long, abort");
      exit(-1);
    }
  }

  pstr += sprintf(pstr, ")");

4789 4790
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

4791
  return (int32_t)strlen(recBuf);
4792 4793
}

4794
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
4795 4796
  char* sampleDataBuf = NULL;

4797
  sampleDataBuf = calloc(
4798
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
4799
  if (sampleDataBuf == NULL) {
4800
      errorPrint("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n",
4801
              __func__, __LINE__,
4802
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE,
4803
              strerror(errno));
4804
      return -1;
4805
  }
4806

4807 4808 4809 4810
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
4811 4812
      errorPrint("%s() LN%d, read sample from csv file failed.\n",
          __func__, __LINE__);
4813
      tmfree(sampleDataBuf);
4814
      superTblInfo->sampleDataBuf = NULL;
4815
      return -1;
H
Hui Li 已提交
4816
  }
4817

4818 4819 4820
  return 0;
}

4821
static int64_t execInsert(threadInfo *pThreadInfo, char *buffer, uint64_t k)
4822 4823
{
  int affectedRows;
4824
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4825

4826 4827
  verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
            __func__, __LINE__, buffer);
4828
  if (superTblInfo) {
4829
    if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) {
4830
      affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false);
4831
    } else if (0 == strncasecmp(superTblInfo->insertMode, "rest", strlen("rest"))) {
4832 4833
      if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port,
                  buffer, NULL /* not set result file */)) {
4834
        affectedRows = -1;
4835 4836
        printf("========restful return fail, threadID[%d]\n",
            pThreadInfo->threadID);
4837 4838 4839
      } else {
        affectedRows = k;
      }
4840 4841 4842 4843
    } else {
      errorPrint("%s() LN%d: unknown insert mode: %s\n",
        __func__, __LINE__, superTblInfo->insertMode);
      affectedRows = 0;
4844
    }
4845
  } else {
4846
    affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false);
4847 4848 4849 4850 4851
  }

  return affectedRows;
}

4852 4853
static void getTableName(char *pTblName,
        threadInfo* pThreadInfo, uint64_t tableSeq)
4854 4855
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4856 4857
  if ((superTblInfo)
          && (AUTO_CREATE_SUBTBL != superTblInfo->autoCreateTable)) {
4858
    if (superTblInfo->childTblLimit > 0) {
4859
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
4860 4861
            superTblInfo->childTblName +
            (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
4862
    } else {
4863

4864
        verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n",
4865 4866
                pThreadInfo->threadID, __func__, __LINE__,
                pThreadInfo->start_table_from,
4867 4868 4869 4870 4871
                pThreadInfo->ntables, tableSeq);
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
    }
  } else {
4872
    snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"",
4873
        g_args.tb_prefix, tableSeq);
4874 4875 4876
  }
}

4877
static int64_t generateDataTail(
4878
        SSuperTable* superTblInfo,
4879
        uint64_t batch, char* buffer, int64_t remainderBufLen, int64_t insertRows,
4880
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) {
4881 4882
  uint64_t len = 0;
  uint32_t ncols_per_record = 1; // count first col ts
4883

4884 4885
  char *pstr = buffer;

4886
  if (superTblInfo == NULL) {
4887
    uint32_t datatypeSeq = 0;
4888 4889 4890 4891
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
4892 4893
  }

4894
  verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch);
4895

4896 4897 4898 4899 4900 4901 4902 4903
  bool tsRand;
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
                  "rand", strlen("rand")))) {
        tsRand = true;
    } else {
        tsRand = false;
  }

4904
  uint64_t k = 0;
4905
  for (k = 0; k < batch;) {
4906
    char data[MAX_DATA_SIZE];
4907 4908
    memset(data, 0, MAX_DATA_SIZE);

4909
    int64_t retLen = 0;
4910

4911
    if (superTblInfo) {
4912 4913 4914 4915 4916 4917 4918 4919 4920
        if (tsRand) {
            retLen = generateRowData(
                    data,
                    startTime + getTSRandTail(
                        superTblInfo->timeStampStep, k,
                        superTblInfo->disorderRatio,
                        superTblInfo->disorderRange),
                    superTblInfo);
        } else {
4921
          retLen = getRowDataFromSample(
4922 4923
                    data,
                    remainderBufLen,
4924
                    startTime + superTblInfo->timeStampStep * k,
4925
                    superTblInfo,
4926
                    pSamplePos);
4927
        }
4928 4929
        if (retLen > remainderBufLen) {
            break;
4930
        }
4931

4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947
        pstr += snprintf(pstr , retLen + 1, "%s", data);
        k++;
        len += retLen;
        remainderBufLen -= retLen;
    } else {
        char **data_type = g_args.datatype;
        int lenOfBinary = g_args.len_of_binary;
        retLen = generateData(data, data_type,
                ncols_per_record,
                startTime + getTSRandTail(
                    DEFAULT_TIMESTAMP_STEP, k,
                    g_args.disorderRatio,
                    g_args.disorderRange),
                lenOfBinary);
        if (len > remainderBufLen)
            break;
4948

4949 4950 4951 4952
        pstr += sprintf(pstr, "%s", data);
        k++;
        len += retLen;
        remainderBufLen -= retLen;
4953 4954
    }

4955
    verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n",
4956
            __func__, __LINE__, len, k, buffer);
4957 4958

    startFrom ++;
4959

4960
    if (startFrom >= insertRows) {
4961
      break;
4962
    }
4963
  }
4964

4965 4966 4967
  *dataLen = len;
  return k;
}
4968

4969
static int generateSQLHead(char *tableName, int32_t tableSeq,
4970 4971
        threadInfo* pThreadInfo, SSuperTable* superTblInfo,
        char *buffer, int remainderBufLen)
4972 4973
{
  int len;
4974 4975 4976 4977

#define HEAD_BUFF_LEN    1024*24  // 16*1024 + (192+32)*2 + insert into ..
  char headBuf[HEAD_BUFF_LEN];

4978 4979 4980 4981
  if (superTblInfo) {
    if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
4982
            tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq);
4983 4984 4985 4986 4987 4988
      } else {
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
      }
      if (NULL == tagsValBuf) {
4989 4990
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n",
            __func__, __LINE__);
4991 4992 4993
        return -1;
      }

4994 4995 4996
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
4997
                  "%s.%s using %s.%s tags %s values",
4998 4999 5000 5001 5002 5003 5004
                  pThreadInfo->db_name,
                  tableName,
                  pThreadInfo->db_name,
                  superTblInfo->sTblName,
                  tagsValBuf);
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
5005 5006 5007
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
5008
                  "%s.%s values",
5009
                  pThreadInfo->db_name,
5010
                  tableName);
5011
    } else {
5012 5013 5014
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
5015
                  "%s.%s values",
5016 5017 5018 5019
                  pThreadInfo->db_name,
                  tableName);
    }
  } else {
5020 5021 5022
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
5023
                  "%s.%s values",
5024 5025 5026 5027
                  pThreadInfo->db_name,
                  tableName);
  }

5028 5029 5030 5031 5032
  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

5033 5034 5035
  return len;
}

5036
static int64_t generateInterlaceDataBuffer(
5037 5038
        char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes,
        uint64_t tableSeq,
5039
        threadInfo *pThreadInfo, char *buffer,
5040
        int64_t insertRows,
5041
        int64_t startTime,
5042
        uint64_t *pRemainderBufLen)
5043
{
5044
  assert(buffer);
5045 5046 5047 5048 5049 5050 5051 5052 5053 5054
  char *pstr = buffer;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo,
            superTblInfo, pstr, *pRemainderBufLen);

  if (headLen <= 0) {
    return 0;
  }
  // generate data buffer
5055
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n",
5056 5057 5058 5059 5060
            pThreadInfo->threadID, __func__, __LINE__, i, buffer);

  pstr += headLen;
  *pRemainderBufLen -= headLen;

5061
  int64_t dataLen = 0;
5062

5063
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%"PRIu64" batchPerTbl = %"PRIu64"\n",
5064 5065 5066 5067 5068 5069 5070 5071 5072 5073
            pThreadInfo->threadID, __func__, __LINE__,
            i, batchPerTblTimes, batchPerTbl);

  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
      startTime = taosGetTimestamp(pThreadInfo->time_precision);
    }
  } else {
      startTime = 1500000000000;
  }
5074

5075
  int64_t k = generateDataTail(
5076
    superTblInfo,
5077 5078 5079 5080
    batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
    startTime,
    &(pThreadInfo->samplePos), &dataLen);

5081
  if (k == batchPerTbl) {
5082 5083 5084
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
  } else {
5085
    debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n",
5086
            __func__, __LINE__, k, batchPerTbl);
5087 5088
    pstr -= headLen;
    pstr[0] = '\0';
5089
    k = 0;
5090 5091 5092 5093 5094
  }

  return k;
}

5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110
static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq,
          int disorderRatio, int disorderRange)
{
    int64_t randTail = timeStampStep * seq;
    if (disorderRatio > 0) {
        int rand_num = taosRandom() % 100;
        if(rand_num < disorderRatio) {
            randTail = (randTail +
                    (taosRandom() % disorderRange + 1)) * (-1);
            debugPrint("rand data generated, back %"PRId64"\n", randTail);
        }
    }

    return randTail;
}

5111
static int64_t generateProgressiveDataBuffer(
5112
        char *tableName,
5113
        int64_t tableSeq,
5114 5115
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
5116
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos,
5117
        int64_t *pRemainderBufLen)
5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  int ncols_per_record = 1; // count first col ts

  if (superTblInfo == NULL) {
    int datatypeSeq = 0;
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
  }

  assert(buffer != NULL);
5132
  char *pstr = buffer;
5133

5134
  int64_t k = 0;
5135

5136
  memset(buffer, 0, *pRemainderBufLen);
5137

5138
  int64_t headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, superTblInfo,
5139
          buffer, *pRemainderBufLen);
5140 5141 5142 5143

  if (headLen <= 0) {
    return 0;
  }
5144
  pstr += headLen;
5145
  *pRemainderBufLen -= headLen;
5146

5147
  int64_t dataLen;
5148
  k = generateDataTail(superTblInfo,
5149
          g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom,
5150
          startTime,
5151
          pSamplePos, &dataLen);
5152

5153 5154
  return k;
}
5155

5156 5157 5158 5159 5160 5161 5162 5163 5164
static void printStatPerThread(threadInfo *pThreadInfo)
{
  fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n",
          pThreadInfo->threadID,
          pThreadInfo->totalInsertRows,
          pThreadInfo->totalAffectedRows,
          (double)(pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay/1000.0)));
}

5165
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
5166 5167
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
         pThreadInfo->threadID, __func__, __LINE__);
5168

5169
  int64_t insertRows;
5170
  uint64_t interlaceRows;
5171

5172
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5173

5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186
  if (superTblInfo) {
    insertRows = superTblInfo->insertRows;

    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    insertRows = g_args.num_of_DPT;
    interlaceRows = g_args.interlace_rows;
  }
5187

5188
  if (interlaceRows > insertRows)
5189
    interlaceRows = insertRows;
5190

5191 5192
  if (interlaceRows > g_args.num_of_RPR)
    interlaceRows = g_args.num_of_RPR;
5193

5194 5195 5196 5197 5198 5199 5200 5201 5202 5203
  int insertMode;

  if (interlaceRows > 0) {
    insertMode = INTERLACE_INSERT_MODE;
  } else {
    insertMode = PROGRESSIVE_INSERT_MODE;
  }

  // TODO: prompt tbl count multple interlace rows and batch
  //
5204

5205
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
5206
  char* buffer = calloc(maxSqlLen, 1);
5207
  if (NULL == buffer) {
5208
    errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5209
              __func__, __LINE__, maxSqlLen, strerror(errno));
5210 5211 5212
    return NULL;
  }

5213
  char tableName[TSDB_TABLE_NAME_LEN];
5214 5215 5216 5217

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

5218
  int64_t nTimeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
5219

5220
  uint64_t insert_interval =
5221
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5222 5223
  uint64_t st = 0;
  uint64_t et = UINT64_MAX;
5224

5225 5226 5227
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5228

5229
  uint64_t tableSeq = pThreadInfo->start_table_from;
5230

5231
  debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n",
5232
          pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from,
5233 5234 5235 5236
          pThreadInfo->ntables, insertRows);

  int64_t startTime = pThreadInfo->start_time;

5237 5238
  uint64_t batchPerTbl = interlaceRows;
  uint64_t batchPerTblTimes;
5239

5240
  if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) {
5241
    batchPerTblTimes =
5242
        g_args.num_of_RPR / interlaceRows;
5243 5244 5245 5246
  } else {
    batchPerTblTimes = 1;
  }

5247
  uint64_t generatedRecPerTbl = 0;
5248
  bool flagSleep = true;
5249
  uint64_t sleepTimeTotal = 0;
5250

5251 5252 5253
  char *strInsertInto = "insert into ";
  int nInsertBufLen = strlen(strInsertInto);

5254
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
5255
    if ((flagSleep) && (insert_interval)) {
5256
        st = taosGetTimestampMs();
5257
        flagSleep = false;
5258 5259
    }
    // generate data
5260
    memset(buffer, 0, maxSqlLen);
5261
    uint64_t remainderBufLen = maxSqlLen;
5262

5263
    char *pstr = buffer;
5264 5265 5266 5267 5268

    int len = snprintf(pstr, nInsertBufLen + 1, "%s", strInsertInto);
    pstr += len;
    remainderBufLen -= len;

5269
    uint64_t recOfBatch = 0;
5270

5271
    for (uint64_t i = 0; i < batchPerTblTimes; i ++) {
5272
      getTableName(tableName, pThreadInfo, tableSeq);
5273 5274 5275
      if (0 == strlen(tableName)) {
        errorPrint("[%d] %s() LN%d, getTableName return null\n",
            pThreadInfo->threadID, __func__, __LINE__);
5276
        free(buffer);
5277 5278
        return NULL;
      }
5279

5280
      uint64_t oldRemainderLen = remainderBufLen;
5281
      int64_t generated = generateInterlaceDataBuffer(
5282 5283 5284 5285
        tableName, batchPerTbl, i, batchPerTblTimes,
        tableSeq,
        pThreadInfo, pstr,
        insertRows,
5286
        startTime,
5287
        &remainderBufLen);
5288

5289 5290
      debugPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5291
      if (generated < 0) {
5292
        errorPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
5293
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5294
        goto free_of_interlace;
5295 5296
      } else if (generated == 0) {
        break;
5297 5298
      }

5299
      tableSeq ++;
5300
      recOfBatch += batchPerTbl;
5301
      pstr += (oldRemainderLen - remainderBufLen);
5302
//      startTime += batchPerTbl * superTblInfo->timeStampStep;
5303
      pThreadInfo->totalInsertRows += batchPerTbl;
5304
      verbosePrint("[%d] %s() LN%d batchPerTbl=%"PRId64" recOfBatch=%"PRId64"\n",
5305 5306
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
5307

5308 5309 5310 5311
      if (insertMode == INTERLACE_INSERT_MODE) {
          if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
            // turn to first table
            tableSeq = pThreadInfo->start_table_from;
5312
            generatedRecPerTbl += batchPerTbl;
5313 5314

            startTime = pThreadInfo->start_time
5315
              + generatedRecPerTbl * nTimeStampStep;
5316

5317 5318 5319 5320
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;

5321 5322 5323 5324
            int remainRows = insertRows - generatedRecPerTbl;
            if ((remainRows > 0) && (batchPerTbl > remainRows))
              batchPerTbl = remainRows;

5325 5326
            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
5327
          }
5328 5329
      }

5330
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n",
5331 5332 5333 5334
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
5335 5336 5337
        break;
    }

5338
    verbosePrint("[%d] %s() LN%d recOfBatch=%"PRIu64" totalInsertRows=%"PRIu64"\n",
5339 5340 5341 5342 5343
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
           pThreadInfo->threadID, __func__, __LINE__, buffer);

5344
    startTs = taosGetTimestampMs();
5345

5346 5347 5348 5349 5350 5351 5352
    if (recOfBatch == 0) {
      errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n",
              pThreadInfo->threadID, __func__, __LINE__,
              recOfBatch);
      errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n");
      goto free_of_interlace;
    }
5353
    int64_t affectedRows = execInsert(pThreadInfo, buffer, recOfBatch);
5354

5355
    endTs = taosGetTimestampMs();
5356 5357
    uint64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n",
5358
            __func__, __LINE__, delay);
5359 5360 5361
    verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5362 5363 5364 5365 5366 5367

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

5368 5369
    if (recOfBatch != affectedRows) {
        errorPrint("[%d] %s() LN%d execInsert insert %"PRIu64", affected rows: %"PRId64"\n%s\n",
5370
                pThreadInfo->threadID, __func__, __LINE__,
5371
                recOfBatch, affectedRows, buffer);
5372
        goto free_of_interlace;
5373
    }
5374

5375
    pThreadInfo->totalAffectedRows += affectedRows;
5376

5377 5378
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
5379
      printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n",
5380 5381 5382
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5383 5384
      lastPrintTime = currentPrintTime;
    }
5385

5386
    if ((insert_interval) && flagSleep) {
5387
      et = taosGetTimestampMs();
5388

5389 5390
      if (insert_interval > (et - st) ) {
        int sleepTime = insert_interval - (et -st);
5391 5392
        performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleepTime);
5393 5394 5395
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
5396
    }
5397 5398
  }

5399
free_of_interlace:
5400
  tmfree(buffer);
5401
  printStatPerThread(pThreadInfo);
5402 5403 5404
  return NULL;
}

5405 5406 5407 5408 5409 5410 5411 5412
// sync insertion
/*
   1 thread: 100 tables * 2000  rows/s
   1 thread: 10  tables * 20000 rows/s
   6 thread: 300 tables * 2000  rows/s

   2 taosinsertdata , 1 thread:  10  tables * 20000 rows/s
*/
5413
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
5414
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
5415

5416
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5417
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
5418

5419
  char* buffer = calloc(maxSqlLen, 1);
5420
  if (NULL == buffer) {
5421
    errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5422
              maxSqlLen,
5423 5424 5425
              strerror(errno));
    return NULL;
  }
5426

5427 5428 5429
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5430

5431
  int64_t timeStampStep =
5432
      superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
5433
/*  int insert_interval =
5434
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5435 5436
  uint64_t st = 0;
  uint64_t et = 0xffffffff;
5437
  */
5438

5439 5440
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
5441

5442
  pThreadInfo->samplePos = 0;
5443

5444
  for (uint64_t tableSeq =
5445
          pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to;
5446
        tableSeq ++) {
5447
    int64_t start_time = pThreadInfo->start_time;
5448

5449
    int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
5450

5451 5452
    verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows);

5453
    for (uint64_t i = 0; i < insertRows;) {
5454
        /*
5455
      if (insert_interval) {
5456
            st = taosGetTimestampMs();
5457
      }
5458
      */
5459

5460 5461
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
5462
      verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n",
5463 5464 5465
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);

5466
      int64_t remainderBufLen = maxSqlLen;
5467 5468 5469 5470 5471 5472 5473 5474
      char *pstr = buffer;
      int nInsertBufLen = strlen("insert into ");

      int len = snprintf(pstr, nInsertBufLen + 1, "%s", "insert into ");

      pstr += len;
      remainderBufLen -= len;

5475
      int64_t generated = generateProgressiveDataBuffer(
5476
              tableName, tableSeq, pThreadInfo, pstr, insertRows,
5477
            i, start_time,
5478 5479
            &(pThreadInfo->samplePos),
            &remainderBufLen);
5480 5481 5482
      if (generated > 0)
        i += generated;
      else
5483
        goto free_of_progressive;
5484

5485
      start_time +=  generated * timeStampStep;
5486
      pThreadInfo->totalInsertRows += generated;
5487

5488
      startTs = taosGetTimestampMs();
5489

5490
      int64_t affectedRows = execInsert(pThreadInfo, buffer, generated);
5491

5492
      endTs = taosGetTimestampMs();
5493
      uint64_t delay = endTs - startTs;
5494 5495
      performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n",
              __func__, __LINE__, delay);
5496 5497 5498
      verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5499

5500 5501 5502 5503
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
5504

5505 5506 5507 5508 5509
      if (affectedRows < 0) {
        errorPrint("%s() LN%d, affected rows: %"PRId64"\n",
                __func__, __LINE__, affectedRows);
        goto free_of_progressive;
      }
5510 5511 5512

      pThreadInfo->totalAffectedRows += affectedRows;

5513 5514
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
5515
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
5516 5517 5518
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5519 5520 5521
        lastPrintTime = currentPrintTime;
      }

5522
      if (i >= insertRows)
5523
        break;
5524
/*
5525
      if (insert_interval) {
5526
        et = taosGetTimestampMs();
5527

5528 5529
        if (insert_interval > ((et - st)) ) {
            int sleep_time = insert_interval - (et -st);
5530 5531
            performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleep_time);
5532 5533
            taosMsleep(sleep_time); // ms
        }
5534
      }
5535
      */
5536 5537
    }   // num_of_DPT

5538 5539
    if (g_args.verbose_print) {
      if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo &&
5540 5541
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
5542
          verbosePrint("%s() LN%d samplePos=%"PRId64"\n",
5543
                  __func__, __LINE__, pThreadInfo->samplePos);
5544
      }
5545
    }
5546
  } // tableSeq
5547

5548
free_of_progressive:
5549
  tmfree(buffer);
5550
  printStatPerThread(pThreadInfo);
5551
  return NULL;
H
Hui Li 已提交
5552 5553
}

5554 5555
static void* syncWrite(void *sarg) {

5556 5557
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5558

5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570
  int interlaceRows;

  if (superTblInfo) {
    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    interlaceRows = g_args.interlace_rows;
  }
5571

5572
  if (interlaceRows > 0) {
5573
    // interlace mode
5574
    return syncWriteInterlace(pThreadInfo);
5575 5576
  } else {
    // progressive mode
5577
    return syncWriteProgressive(pThreadInfo);
5578
  }
5579

5580 5581
}

5582
static void callBack(void *param, TAOS_RES *res, int code) {
5583 5584
  threadInfo* pThreadInfo = (threadInfo*)param;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5585

5586 5587
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5588
  if (insert_interval) {
5589 5590 5591
    pThreadInfo->et = taosGetTimestampMs();
    if ((pThreadInfo->et - pThreadInfo->st) < insert_interval) {
      taosMsleep(insert_interval - (pThreadInfo->et - pThreadInfo->st)); // ms
5592
    }
H
Hui Li 已提交
5593
  }
5594

5595
  char *buffer = calloc(1, pThreadInfo->superTblInfo->maxSqlLen);
5596
  char data[MAX_DATA_SIZE];
5597
  char *pstr = buffer;
5598 5599
  pstr += sprintf(pstr, "insert into %s.%s%"PRId64" values",
          pThreadInfo->db_name, pThreadInfo->tb_prefix,
5600 5601 5602 5603 5604 5605 5606 5607
          pThreadInfo->start_table_from);
//  if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
  if (pThreadInfo->counter >= g_args.num_of_RPR) {
    pThreadInfo->start_table_from++;
    pThreadInfo->counter = 0;
  }
  if (pThreadInfo->start_table_from > pThreadInfo->end_table_to) {
    tsem_post(&pThreadInfo->lock_sem);
5608 5609 5610 5611
    free(buffer);
    taos_free_result(res);
    return;
  }
5612

5613
  for (int i = 0; i < g_args.num_of_RPR; i++) {
5614
    int rand_num = taosRandom() % 100;
5615 5616
    if (0 != pThreadInfo->superTblInfo->disorderRatio
            && rand_num < pThreadInfo->superTblInfo->disorderRatio) {
5617 5618
      int64_t d = pThreadInfo->lastTs
          - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1);
5619
      generateRowData(data, d, pThreadInfo->superTblInfo);
5620
    } else {
5621
      generateRowData(data, pThreadInfo->lastTs += 1000, pThreadInfo->superTblInfo);
H
Hui Li 已提交
5622
    }
5623
    pstr += sprintf(pstr, "%s", data);
5624
    pThreadInfo->counter++;
H
Hui Li 已提交
5625

5626
    if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
5627
      break;
H
Hui Li 已提交
5628 5629
    }
  }
5630

5631
  if (insert_interval) {
5632
    pThreadInfo->st = taosGetTimestampMs();
5633
  }
5634
  taos_query_a(pThreadInfo->taos, buffer, callBack, pThreadInfo);
5635
  free(buffer);
H
Hui Li 已提交
5636

5637
  taos_free_result(res);
H
Hui Li 已提交
5638 5639
}

5640
static void *asyncWrite(void *sarg) {
5641 5642
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5643

5644 5645 5646
  pThreadInfo->st = 0;
  pThreadInfo->et = 0;
  pThreadInfo->lastTs = pThreadInfo->start_time;
5647

5648
  int insert_interval =
5649
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5650
  if (insert_interval) {
5651
    pThreadInfo->st = taosGetTimestampMs();
H
Hui Li 已提交
5652
  }
5653
  taos_query_a(pThreadInfo->taos, "show databases", callBack, pThreadInfo);
5654

5655
  tsem_wait(&(pThreadInfo->lock_sem));
H
Hui Li 已提交
5656 5657 5658 5659

  return NULL;
}

5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685
static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in *serv_addr)
{
  uint16_t rest_port = port + TSDB_PORT_HTTP;
  struct hostent *server = gethostbyname(host);
  if ((server == NULL) || (server->h_addr == NULL)) {
    errorPrint("%s", "ERROR, no such host");
    return -1;
  }

  debugPrint("h_name: %s\nh_addr=%p\nh_addretype: %s\nh_length: %d\n",
            server->h_name,
            server->h_addr,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

  memset(serv_addr, 0, sizeof(struct sockaddr_in));
  serv_addr->sin_family = AF_INET;
  serv_addr->sin_port = htons(rest_port);
#ifdef WINDOWS
  serv_addr->sin_addr.s_addr = inet_addr(host);
#else
  memcpy(&(serv_addr->sin_addr.s_addr), server->h_addr, server->h_length);
#endif
  return 0;
}

5686 5687
static void startMultiThreadInsertData(int threads, char* db_name,
        char* precision,SSuperTable* superTblInfo) {
5688

5689 5690
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  assert(pids != NULL);
H
Hui Li 已提交
5691

5692 5693
  threadInfo *infos = malloc(threads * sizeof(threadInfo));
  assert(infos != NULL);
5694

5695 5696
  memset(pids, 0, threads * sizeof(pthread_t));
  memset(infos, 0, threads * sizeof(threadInfo));
5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713

  //TAOS* taos;
  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
  //  if (NULL == taos) {
  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
  //    exit(-1);
  //  }
  //}

  int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
  if (0 != precision[0]) {
    if (0 == strncasecmp(precision, "ms", 2)) {
      timePrec = TSDB_TIME_PRECISION_MILLI;
    }  else if (0 == strncasecmp(precision, "us", 2)) {
      timePrec = TSDB_TIME_PRECISION_MICRO;
    }  else {
5714
      errorPrint("Not support precision: %s\n", precision);
5715 5716 5717 5718
      exit(-1);
    }
  }

5719
  int64_t start_time;
5720 5721 5722
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
        start_time = taosGetTimestamp(timePrec);
5723
    } else {
5724 5725 5726 5727 5728
      if (TSDB_CODE_SUCCESS != taosParseTime(
        superTblInfo->startTimestamp,
        &start_time,
        strlen(superTblInfo->startTimestamp),
        timePrec, 0)) {
5729
          ERROR_EXIT("failed to parse time!\n");
5730
      }
5731
    }
5732 5733
  } else {
     start_time = 1500000000000;
5734 5735
  }

5736
  int64_t start = taosGetTimestampMs();
5737

5738
  // read sample data from file first
5739
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5740
              "sample", strlen("sample")))) {
5741
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5742 5743
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
              __func__, __LINE__);
5744 5745 5746 5747
      exit(-1);
    }
  }

5748
  // read sample data from file first
5749
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5750 5751
              "sample", strlen("sample")))) {
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5752 5753
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
              __func__, __LINE__);
5754 5755 5756 5757 5758 5759 5760 5761
      exit(-1);
    }
  }

  TAOS* taos = taos_connect(
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
  if (NULL == taos) {
5762 5763
    errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
                __func__, __LINE__, taos_errstr(NULL));
5764 5765 5766
    exit(-1);
  }

5767
  int64_t ntables = 0;
5768
  uint64_t startFrom;
5769

5770
  if (superTblInfo) {
5771 5772
    int64_t limit;
    uint64_t offset;
5773

5774
    if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) &&
5775
            ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) {
5776
      printf("WARNING: offset and limit will not be used since the child tables not exists!\n");
5777 5778
    }

5779
    if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) {
5780 5781
      if ((superTblInfo->childTblLimit < 0)
          || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit)
5782
            > (superTblInfo->childTblCount))) {
5783 5784 5785 5786 5787 5788
        superTblInfo->childTblLimit =
            superTblInfo->childTblCount - superTblInfo->childTblOffset;
      }

      offset = superTblInfo->childTblOffset;
      limit = superTblInfo->childTblLimit;
5789
    } else {
5790 5791
      limit = superTblInfo->childTblCount;
      offset = 0;
5792 5793
    }

5794 5795 5796
    ntables = limit;
    startFrom = offset;

5797 5798 5799 5800
    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
        && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
            > superTblInfo->childTblCount)) {
      printf("WARNING: specified offset + limit > child table count!\n");
5801
      prompt();
5802 5803 5804 5805 5806
    }

    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
            && (0 == superTblInfo->childTblLimit)) {
      printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
5807
      prompt();
5808 5809
    }

5810 5811 5812
    superTblInfo->childTblName = (char*)calloc(1,
        limit * TSDB_TABLE_NAME_LEN);
    if (superTblInfo->childTblName == NULL) {
5813
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
5814 5815 5816 5817
      taos_close(taos);
      exit(-1);
    }

5818
    int64_t childTblCount;
5819 5820 5821 5822 5823 5824
    getChildNameOfSuperTableWithLimitAndOffset(
        taos,
        db_name, superTblInfo->sTblName,
        &superTblInfo->childTblName, &childTblCount,
        limit,
        offset);
5825 5826 5827
  } else {
    ntables = g_args.num_of_tables;
    startFrom = 0;
5828
  }
5829

5830 5831
  taos_close(taos);

5832
  int64_t a = ntables / threads;
5833 5834 5835 5836 5837
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

5838
  int64_t b = 0;
5839 5840 5841 5842
  if (threads != 0) {
    b = ntables % threads;
  }

5843 5844 5845 5846 5847 5848
  if ((superTblInfo)
      && (0 == strncasecmp(superTblInfo->insertMode, "rest", strlen("rest")))) {
    if (convertHostToServAddr(g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0)
      exit(-1);
  }

H
Hui Li 已提交
5849
  for (int i = 0; i < threads; i++) {
5850 5851 5852 5853 5854
    threadInfo *pThreadInfo = infos + i;
    pThreadInfo->threadID = i;
    tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE);
    pThreadInfo->time_precision = timePrec;
    pThreadInfo->superTblInfo = superTblInfo;
5855

5856 5857
    pThreadInfo->start_time = start_time;
    pThreadInfo->minDelay = UINT64_MAX;
5858

5859 5860
    if ((NULL == superTblInfo) ||
            (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) {
5861 5862
      //pThreadInfo->taos = taos;
      pThreadInfo->taos = taos_connect(
5863
              g_Dbs.host, g_Dbs.user,
5864
              g_Dbs.password, db_name, g_Dbs.port);
5865
      if (NULL == pThreadInfo->taos) {
5866 5867
        errorPrint(
                "connect to server fail from insert sub thread, reason: %s\n",
5868
                taos_errstr(NULL));
5869 5870 5871
        exit(-1);
      }
    } else {
5872
      pThreadInfo->taos = NULL;
5873 5874
    }

5875
/*    if ((NULL == superTblInfo)
5876
            || (0 == superTblInfo->multiThreadWriteOneTbl)) {
5877
            */
5878 5879 5880 5881
      pThreadInfo->start_table_from = startFrom;
      pThreadInfo->ntables = i<b?a+1:a;
      pThreadInfo->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = pThreadInfo->end_table_to + 1;
5882
/*    } else {
5883 5884 5885
      pThreadInfo->start_table_from = 0;
      pThreadInfo->ntables = superTblInfo->childTblCount;
      pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint();
5886
    }
5887
*/
5888
    tsem_init(&(pThreadInfo->lock_sem), 0, 0);
5889
    if (ASYNC_MODE == g_Dbs.asyncMode) {
5890
      pthread_create(pids + i, NULL, asyncWrite, pThreadInfo);
5891
    } else {
5892
      pthread_create(pids + i, NULL, syncWrite, pThreadInfo);
5893
    }
H
Hui Li 已提交
5894
  }
5895

H
Hui Li 已提交
5896 5897 5898 5899
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

5900 5901 5902 5903
  uint64_t totalDelay = 0;
  uint64_t maxDelay = 0;
  uint64_t minDelay = UINT64_MAX;
  uint64_t cntDelay = 1;
5904 5905
  double  avgDelay = 0;

H
Hui Li 已提交
5906
  for (int i = 0; i < threads; i++) {
5907
    threadInfo *pThreadInfo = infos + i;
5908

5909 5910
    tsem_destroy(&(pThreadInfo->lock_sem));
    taos_close(pThreadInfo->taos);
5911

5912
    debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n",
5913
            __func__, __LINE__,
5914 5915
            pThreadInfo->threadID, pThreadInfo->totalInsertRows,
            pThreadInfo->totalAffectedRows);
5916
    if (superTblInfo) {
5917 5918
        superTblInfo->totalAffectedRows += pThreadInfo->totalAffectedRows;
        superTblInfo->totalInsertRows += pThreadInfo->totalInsertRows;
5919
    } else {
5920 5921
        g_args.totalAffectedRows += pThreadInfo->totalAffectedRows;
        g_args.totalInsertRows += pThreadInfo->totalInsertRows;
5922
    }
5923

5924 5925 5926 5927
    totalDelay  += pThreadInfo->totalDelay;
    cntDelay   += pThreadInfo->cntDelay;
    if (pThreadInfo->maxDelay > maxDelay) maxDelay = pThreadInfo->maxDelay;
    if (pThreadInfo->minDelay < minDelay) minDelay = pThreadInfo->minDelay;
H
Hui Li 已提交
5928
  }
5929
  cntDelay -= 1;
H
Hui Li 已提交
5930

5931
  if (cntDelay == 0)    cntDelay = 1;
5932 5933
  avgDelay = (double)totalDelay / cntDelay;

5934 5935
  int64_t end = taosGetTimestampMs();
  int64_t t = end - start;
5936 5937

  if (superTblInfo) {
5938
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
5939
          t / 1000.0, superTblInfo->totalInsertRows,
5940 5941
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5942
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
5943 5944 5945 5946

    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
5947
          t / 1000.0, superTblInfo->totalInsertRows,
5948 5949
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5950
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
5951
    }
5952
  } else {
5953
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
5954
          t / 1000.0, g_args.totalInsertRows,
5955 5956
          g_args.totalAffectedRows,
          threads, db_name,
5957
          (double)g_args.totalInsertRows / (t / 1000.0));
5958 5959 5960
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
5961
          t * 1000.0, g_args.totalInsertRows,
5962 5963
          g_args.totalAffectedRows,
          threads, db_name,
5964
          (double)g_args.totalInsertRows / (t / 1000.0));
5965
    }
5966
  }
5967

5968
  fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
5969
          avgDelay, maxDelay, minDelay);
5970 5971
  if (g_fpOfInsertResult) {
    fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
5972
          avgDelay, maxDelay, minDelay);
5973
  }
5974

5975 5976
  //taos_close(taos);

H
Hui Li 已提交
5977
  free(pids);
5978
  free(infos);
H
Hui Li 已提交
5979 5980
}

5981
static void *readTable(void *sarg) {
5982
#if 1
5983 5984
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  TAOS *taos = pThreadInfo->taos;
H
hzcheng 已提交
5985
  char command[BUFFER_SIZE] = "\0";
5986 5987 5988
  uint64_t sTime = pThreadInfo->start_time;
  char *tb_prefix = pThreadInfo->tb_prefix;
  FILE *fp = fopen(pThreadInfo->filePath, "a");
H
Hui Li 已提交
5989
  if (NULL == fp) {
5990
    errorPrint( "fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno));
H
Hui Li 已提交
5991 5992
    return NULL;
  }
5993

5994
  int64_t num_of_DPT;
5995 5996
/*  if (pThreadInfo->superTblInfo) {
    num_of_DPT = pThreadInfo->superTblInfo->insertRows; //  nrecords_per_table;
5997
  } else {
5998
  */
5999
      num_of_DPT = g_args.num_of_DPT;
6000
//  }
6001

6002
  int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
6003
  int64_t totalData = num_of_DPT * num_of_tables;
6004
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6005 6006 6007 6008 6009

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
6010
  printf("%"PRId64" records:\n", totalData);
H
hzcheng 已提交
6011 6012
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

6013
  for (int j = 0; j < n; j++) {
H
hzcheng 已提交
6014
    double totalT = 0;
6015
    uint64_t count = 0;
6016 6017
    for (int64_t i = 0; i < num_of_tables; i++) {
      sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64,
6018
              aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
6019

6020
      double t = taosGetTimestampMs();
S
Shuaiqiang Chang 已提交
6021 6022
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
6023

S
Shuaiqiang Chang 已提交
6024
      if (code != 0) {
6025
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6026
        taos_free_result(pSql);
H
hzcheng 已提交
6027
        taos_close(taos);
6028
        fclose(fp);
6029
        return NULL;
H
hzcheng 已提交
6030 6031
      }

6032
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6033 6034 6035
        count++;
      }

6036
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6037 6038
      totalT += t;

S
Shuaiqiang Chang 已提交
6039
      taos_free_result(pSql);
H
hzcheng 已提交
6040 6041
    }

6042
    fprintf(fp, "|%10s  |   %"PRId64"   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
6043
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
6044
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
6045
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000);
H
hzcheng 已提交
6046 6047 6048
  }
  fprintf(fp, "\n");
  fclose(fp);
6049
#endif
H
hzcheng 已提交
6050 6051 6052
  return NULL;
}

6053
static void *readMetric(void *sarg) {
6054
#if 1
6055 6056
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  TAOS *taos = pThreadInfo->taos;
H
hzcheng 已提交
6057
  char command[BUFFER_SIZE] = "\0";
6058
  FILE *fp = fopen(pThreadInfo->filePath, "a");
H
Hui Li 已提交
6059
  if (NULL == fp) {
6060
    printf("fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno));
H
Hui Li 已提交
6061 6062
    return NULL;
  }
6063

6064 6065
  int64_t num_of_DPT = pThreadInfo->superTblInfo->insertRows;
  int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
6066
  int64_t totalData = num_of_DPT * num_of_tables;
6067
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6068 6069 6070 6071 6072

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
6073 6074
  printf("%"PRId64" records:\n", totalData);
  fprintf(fp, "Querying On %"PRId64" records:\n", totalData);
H
hzcheng 已提交
6075 6076

  for (int j = 0; j < n; j++) {
6077
    char condition[COND_BUF_LEN] = "\0";
B
Bomin Zhang 已提交
6078
    char tempS[64] = "\0";
H
hzcheng 已提交
6079

6080
    int64_t m = 10 < num_of_tables ? 10 : num_of_tables;
H
hzcheng 已提交
6081

6082
    for (int64_t i = 1; i <= m; i++) {
H
hzcheng 已提交
6083
      if (i == 1) {
6084
        sprintf(tempS, "t1 = %"PRId64"", i);
H
hzcheng 已提交
6085
      } else {
6086
        sprintf(tempS, " or t1 = %"PRId64" ", i);
H
hzcheng 已提交
6087
      }
6088
      strncat(condition, tempS, COND_BUF_LEN - 1);
H
hzcheng 已提交
6089

L
liu0x54 已提交
6090
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
6091 6092 6093 6094

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

6095
      double t = taosGetTimestampMs();
H
hzcheng 已提交
6096

S
Shuaiqiang Chang 已提交
6097 6098 6099 6100
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
6101
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6102
        taos_free_result(pSql);
H
hzcheng 已提交
6103
        taos_close(taos);
6104
        fclose(fp);
6105
        return NULL;
H
hzcheng 已提交
6106 6107
      }
      int count = 0;
6108
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6109 6110
        count++;
      }
6111
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6112

6113
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
6114 6115
              num_of_tables * num_of_DPT / (t * 1000.0), t);
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0);
H
hzcheng 已提交
6116

S
Shuaiqiang Chang 已提交
6117
      taos_free_result(pSql);
H
hzcheng 已提交
6118 6119 6120 6121
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
6122
#endif
H
hzcheng 已提交
6123 6124 6125
  return NULL;
}

6126 6127 6128
static void prompt()
{
  if (!g_args.answer_yes) {
6129
    printf("         Press enter key to continue or Ctrl-C to stop\n\n");
6130 6131 6132 6133
    (void)getchar();
  }
}

6134
static int insertTestProcess() {
6135

6136 6137 6138
  setupForAnsiEscape();
  int ret = printfInsertMeta();
  resetAfterAnsiEscape();
6139

6140 6141 6142
  if (ret == -1)
    exit(EXIT_FAILURE);

6143
  debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
6144 6145
  g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
  if (NULL == g_fpOfInsertResult) {
6146
    errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
6147 6148
    return -1;
  }
6149

6150 6151
  if (g_fpOfInsertResult)
    printfInsertMetaToFile(g_fpOfInsertResult);
6152

6153
  prompt();
6154

6155 6156 6157
  init_rand_data();

  // create database and super tables
6158
  if(createDatabasesAndStables() != 0) {
6159 6160
    if (g_fpOfInsertResult)
      fclose(g_fpOfInsertResult);
6161 6162
    return -1;
  }
6163 6164

  // pretreatement
6165
  prepareSampleData();
6166

6167 6168 6169 6170
  double start;
  double end;

  // create child tables
6171
  start = taosGetTimestampMs();
6172
  createChildTables();
6173
  end = taosGetTimestampMs();
6174

6175
  if (g_totalChildTables > 0) {
6176
    fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6177
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6178 6179
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
6180
            "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6181
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6182
    }
6183
  }
6184

6185
  taosMsleep(1000);
6186
  // create sub threads for inserting data
6187
  //start = taosGetTimestampMs();
6188
  for (int i = 0; i < g_Dbs.dbCount; i++) {
6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
        for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {

          SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];

          if (superTblInfo && (superTblInfo->insertRows > 0)) {
            startMultiThreadInsertData(
              g_Dbs.threadCount,
              g_Dbs.db[i].dbName,
              g_Dbs.db[i].dbCfg.precision,
              superTblInfo);
          }
6202
        }
6203
      }
6204 6205
    } else {
        startMultiThreadInsertData(
6206 6207 6208
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
6209
          NULL);
H
Hui Li 已提交
6210
    }
6211
  }
6212
  //end = taosGetTimestampMs();
6213

6214
  //int64_t    totalInsertRows = 0;
6215
  //int64_t    totalAffectedRows = 0;
6216
  //for (int i = 0; i < g_Dbs.dbCount; i++) {
6217
  //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6218
  //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
6219 6220
  //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
  //}
6221
  //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
6222
  postFreeResource();
6223

6224 6225 6226
  return 0;
}

6227 6228
static void *specifiedTableQuery(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6229

6230
  if (pThreadInfo->taos == NULL) {
6231 6232 6233 6234 6235 6236 6237 6238
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6239
            pThreadInfo->threadID, taos_errstr(NULL));
6240 6241
      return NULL;
    } else {
6242
      pThreadInfo->taos = taos;
6243 6244 6245
    }
  }

6246 6247
  char sqlStr[MAX_DB_NAME_SIZE + 5];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6248 6249
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6250 6251 6252 6253
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
    return NULL;
  }
6254

6255 6256
  uint64_t st = 0;
  uint64_t et = 0;
6257

6258
  uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes;
6259

6260 6261 6262
  uint64_t totalQueried = 0;
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
6263

6264
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') {
6265
    sprintf(pThreadInfo->filePath, "%s-%d",
6266 6267 6268 6269
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }

6270
  while(queryTimes --) {
6271 6272 6273
    if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) <
            (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms
6274 6275
    }

6276 6277 6278
    st = taosGetTimestampMs();

    selectAndGetResult(pThreadInfo,
6279
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]);
6280

6281 6282 6283 6284
    et = taosGetTimestampMs();
    printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n",
              taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0);

6285 6286
    totalQueried ++;
    g_queryInfo.specifiedQueryInfo.totalQueried ++;
6287

6288 6289
    uint64_t  currentPrintTime = taosGetTimestampMs();
    uint64_t  endTs = taosGetTimestampMs();
6290
    if (currentPrintTime - lastPrintTime > 30*1000) {
6291
      debugPrint("%s() LN%d, endTs=%"PRIu64"ms, startTs=%"PRIu64"ms\n",
6292
          __func__, __LINE__, endTs, startTs);
6293
      printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.6f\n",
6294 6295
                    pThreadInfo->threadID,
                    totalQueried,
6296
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6297
      lastPrintTime = currentPrintTime;
6298
    }
H
Hui Li 已提交
6299
  }
6300 6301
  return NULL;
}
H
Hui Li 已提交
6302

6303
static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) {
6304 6305
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
6306 6307
  sprintf(subTblName, "%s.%s",
          g_queryInfo.dbName,
6308
          g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
6309 6310

  //printf("inSql: %s\n", inSql);
6311

6312 6313
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
6314
    return;
H
Hui Li 已提交
6315
  }
6316

6317
  tstrncpy(outSql, inSql, pos - inSql + 1);
6318
  //printf("1: %s\n", outSql);
6319
  strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1);
6320
  //printf("2: %s\n", outSql);
6321
  strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1);
6322
  //printf("3: %s\n", outSql);
H
Hui Li 已提交
6323 6324
}

6325
static void *superTableQuery(void *sarg) {
6326
  char sqlstr[MAX_QUERY_SQL_LENGTH];
6327
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6328

6329
  if (pThreadInfo->taos == NULL) {
6330 6331 6332 6333 6334 6335 6336 6337
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6338
            pThreadInfo->threadID, taos_errstr(NULL));
6339 6340
      return NULL;
    } else {
6341
      pThreadInfo->taos = taos;
6342 6343 6344
    }
  }

6345 6346
  uint64_t st = 0;
  uint64_t et = (int64_t)g_queryInfo.superQueryInfo.queryInterval;
6347

6348 6349 6350
  uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes;
  uint64_t totalQueried = 0;
  uint64_t  startTs = taosGetTimestampMs();
6351

6352
  uint64_t  lastPrintTime = taosGetTimestampMs();
6353
  while(queryTimes --) {
6354 6355 6356
    if (g_queryInfo.superQueryInfo.queryInterval
            && (et - st) < (int64_t)g_queryInfo.superQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.superQueryInfo.queryInterval - (et - st)); // ms
6357
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to);
6358
    }
H
Hui Li 已提交
6359

6360
    st = taosGetTimestampMs();
6361
    for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) {
6362
      for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
6363
        memset(sqlstr,0,sizeof(sqlstr));
6364
        replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i);
6365
        if (g_queryInfo.superQueryInfo.result[j][0] != '\0') {
6366
          sprintf(pThreadInfo->filePath, "%s-%d",
6367
                  g_queryInfo.superQueryInfo.result[j],
6368
                  pThreadInfo->threadID);
6369
        }
6370
        selectAndGetResult(pThreadInfo, sqlstr);
6371 6372 6373 6374 6375 6376 6377

        totalQueried++;
        g_queryInfo.superQueryInfo.totalQueried ++;

        int64_t  currentPrintTime = taosGetTimestampMs();
        int64_t  endTs = taosGetTimestampMs();
        if (currentPrintTime - lastPrintTime > 30*1000) {
6378
          printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.3f\n",
6379 6380
                    pThreadInfo->threadID,
                    totalQueried,
6381
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6382
          lastPrintTime = currentPrintTime;
6383
        }
6384
      }
H
Hui Li 已提交
6385
    }
6386
    et = taosGetTimestampMs();
6387
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%"PRIu64" - %"PRIu64"] once queries duration:%.4fs\n\n",
6388
            taosGetSelfPthreadId(),
6389 6390
            pThreadInfo->start_table_from,
            pThreadInfo->end_table_to,
6391
            (double)(et - st)/1000.0);
6392
  }
6393

6394 6395 6396
  return NULL;
}

6397
static int queryTestProcess() {
6398 6399 6400 6401

  setupForAnsiEscape();
  printfQueryMeta();
  resetAfterAnsiEscape();
6402 6403 6404 6405 6406 6407

  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
6408
          g_queryInfo.port);
6409
  if (taos == NULL) {
6410 6411
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6412 6413 6414
    exit(-1);
  }

6415
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6416 6417
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6418 6419 6420
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6421
  }
6422

6423
  prompt();
6424

6425 6426 6427
  if (g_args.debug_print || g_args.verbose_print) {
    printfQuerySystemInfo(taos);
  }
6428

6429 6430 6431 6432 6433 6434
  if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
    if (convertHostToServAddr(
        g_queryInfo.host, g_queryInfo.port, &g_queryInfo.serv_addr) != 0)
      exit(-1);
  }

6435 6436 6437
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
6438
  int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent;
6439
  uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount;
6440

6441
  uint64_t startTs = taosGetTimestampMs();
6442

6443 6444 6445 6446 6447 6448
  if ((nSqlCount > 0) && (nConcurrent > 0)) {

    pids  = malloc(nConcurrent * nSqlCount * sizeof(pthread_t));
    infos = malloc(nConcurrent * nSqlCount * sizeof(threadInfo));

    if ((NULL == pids) || (NULL == infos)) {
6449
      taos_close(taos);
6450
      ERROR_EXIT("memory allocation failed for create threads\n");
6451
    }
6452

6453 6454 6455
    for (uint64_t i = 0; i < nSqlCount; i++) {
        for (int j = 0; j < nConcurrent; j++) {
            uint64_t seq = i * nConcurrent + j;
6456 6457 6458
            threadInfo *pThreadInfo = infos + seq;
            pThreadInfo->threadID = seq;
            pThreadInfo->querySeq = i;
6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472

            if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {

                char sqlStr[MAX_TB_NAME_SIZE*2];
                sprintf(sqlStr, "use %s", g_queryInfo.dbName);
                if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) {
                    taos_close(taos);
                    free(infos);
                    free(pids);
                    errorPrint( "use database %s failed!\n\n",
                        g_queryInfo.dbName);
                    return -1;
                }
            }
6473

6474
            pThreadInfo->taos = NULL;// TODO: workaround to use separate taos connection;
6475

6476
            pthread_create(pids + seq, NULL, specifiedTableQuery,
6477
                pThreadInfo);
6478
        }
6479
    }
6480
  } else {
6481
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
6482
  }
6483

6484 6485
  taos_close(taos);

6486 6487 6488
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
6489 6490 6491 6492
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
    pidsOfSub  = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t));
    infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo));
6493 6494

    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
6495 6496
      free(infos);
      free(pids);
6497

6498
      ERROR_EXIT("memory allocation failed for create threads\n");
6499
    }
6500

6501
    int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
6502
    int threads = g_queryInfo.superQueryInfo.threadCnt;
6503

6504
    int64_t a = ntables / threads;
6505 6506 6507 6508
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
6509

6510
    int64_t b = 0;
6511 6512 6513
    if (threads != 0) {
      b = ntables % threads;
    }
6514

6515
    uint64_t startFrom = 0;
6516
    for (int i = 0; i < threads; i++) {
6517 6518
      threadInfo *pThreadInfo = infosOfSub + i;
      pThreadInfo->threadID = i;
6519

6520 6521 6522 6523 6524 6525
      pThreadInfo->start_table_from = startFrom;
      pThreadInfo->ntables = i<b?a+1:a;
      pThreadInfo->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = pThreadInfo->end_table_to + 1;
      pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection;
      pthread_create(pidsOfSub + i, NULL, superTableQuery, pThreadInfo);
6526 6527
    }

6528
    g_queryInfo.superQueryInfo.threadCnt = threads;
6529
  } else {
6530
    g_queryInfo.superQueryInfo.threadCnt = 0;
6531
  }
6532

6533 6534 6535 6536 6537 6538
  if ((nSqlCount > 0) && (nConcurrent > 0)) {
    for (int i = 0; i < nConcurrent; i++) {
      for (int j = 0; j < nSqlCount; j++) {
        pthread_join(pids[i * nSqlCount + j], NULL);
      }
    }
6539
  }
S
Shuaiqiang Chang 已提交
6540

6541
  tmfree((char*)pids);
6542
  tmfree((char*)infos);
6543

6544
  for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
6545
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
6546
  }
H
Hui Li 已提交
6547

6548
  tmfree((char*)pidsOfSub);
6549
  tmfree((char*)infosOfSub);
6550

6551
//  taos_close(taos);// TODO: workaround to use separate taos connection;
6552
  uint64_t endTs = taosGetTimestampMs();
6553

6554
  uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried +
6555 6556
    g_queryInfo.superQueryInfo.totalQueried;

6557
  fprintf(stderr, "==== completed total queries: %"PRIu64", the QPS of all threads: %10.3f====\n",
6558
          totalQueried,
6559
          (double)(totalQueried/((endTs-startTs)/1000.0)));
6560 6561 6562
  return 0;
}

6563 6564 6565 6566 6567 6568 6569 6570 6571
static void stable_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
  if (res == NULL || taos_errno(res) != 0) {
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
    return;
  }

  if (param)
6572
    fetchResult(res, (threadInfo *)param);
6573 6574 6575 6576 6577
  // tao_unscribe() will free result.
}

static void specified_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
6578
  if (res == NULL || taos_errno(res) != 0) {
6579 6580
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
6581 6582
    return;
  }
6583

6584
  if (param)
6585
    fetchResult(res, (threadInfo *)param);
6586
  // tao_unscribe() will free result.
H
hzcheng 已提交
6587 6588
}

6589
static TAOS_SUB* subscribeImpl(
6590
        QUERY_CLASS class,
6591
        threadInfo *pThreadInfo,
6592
        char *sql, char* topic, bool restart, uint64_t interval)
6593
{
6594
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
6595

6596 6597
  if ((SPECIFIED_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) {
6598 6599
    tsub = taos_subscribe(
            pThreadInfo->taos,
6600
            restart,
6601
            topic, sql, specified_sub_callback, (void*)pThreadInfo,
6602
            g_queryInfo.specifiedQueryInfo.subscribeInterval);
6603 6604
  } else if ((STABLE_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) {
6605 6606 6607 6608 6609
    tsub = taos_subscribe(
            pThreadInfo->taos,
            restart,
            topic, sql, stable_sub_callback, (void*)pThreadInfo,
            g_queryInfo.superQueryInfo.subscribeInterval);
6610
  } else {
6611 6612
    tsub = taos_subscribe(
            pThreadInfo->taos,
6613
            restart,
6614
            topic, sql, NULL, NULL, interval);
6615
  }
6616

6617 6618 6619
  if (tsub == NULL) {
    printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
    return NULL;
6620
  }
6621

6622 6623
  return tsub;
}
H
hzcheng 已提交
6624

6625 6626
static void *superSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6627
  char subSqlstr[MAX_QUERY_SQL_LENGTH];
6628
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT] = {0};
6629
  uint64_t tsubSeq;
H
hzcheng 已提交
6630

6631 6632
  if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
      errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n",
6633 6634 6635 6636 6637
              pThreadInfo->ntables,
              MAX_QUERY_SQL_COUNT);
      exit(-1);
  }

6638
  if (pThreadInfo->taos == NULL) {
6639
    pThreadInfo->taos = taos_connect(g_queryInfo.host,
6640 6641 6642 6643
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
6644
    if (pThreadInfo->taos == NULL) {
6645
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6646
            pThreadInfo->threadID, taos_errstr(NULL));
6647 6648 6649 6650
      return NULL;
    }
  }

6651 6652
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6653 6654
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6655 6656
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
6657 6658
    return NULL;
  }
6659

6660 6661 6662
  char topic[32] = {0};
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6663 6664

      tsubSeq = i - pThreadInfo->start_table_from;
6665 6666 6667 6668 6669
      verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n",
              __func__, __LINE__,
              pThreadInfo->threadID,
              pThreadInfo->start_table_from,
              pThreadInfo->end_table_to, i);
6670 6671
      sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"",
              i, pThreadInfo->querySeq);
6672
      memset(subSqlstr, 0, sizeof(subSqlstr));
6673 6674 6675 6676
      replaceChildTblName(
              g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq],
              subSqlstr, i);
      if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6677
        sprintf(pThreadInfo->filePath, "%s-%d",
6678 6679 6680 6681 6682 6683
                g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
      }

      debugPrint("%s() LN%d, [%d] subSqlstr: %s\n",
              __func__, __LINE__, pThreadInfo->threadID, subSqlstr);
6684
      tsub[tsubSeq] = subscribeImpl(
6685
              STABLE_CLASS,
6686
              pThreadInfo, subSqlstr, topic,
6687 6688
              g_queryInfo.superQueryInfo.subscribeRestart,
              g_queryInfo.superQueryInfo.subscribeInterval);
6689
      if (NULL == tsub[tsubSeq]) {
6690
        taos_close(pThreadInfo->taos);
6691 6692
        return NULL;
      }
6693
  }
6694 6695

  // start loop to consume result
6696 6697 6698 6699
  int consumed[MAX_QUERY_SQL_COUNT];
  for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) {
    consumed[i] = 0;
  }
6700
  TAOS_RES* res = NULL;
6701

6702 6703
  uint64_t st = 0, et = 0;

6704 6705 6706 6707
  while ((g_queryInfo.superQueryInfo.endAfterConsume == -1)
          || (g_queryInfo.superQueryInfo.endAfterConsume <
              consumed[pThreadInfo->end_table_to - pThreadInfo->start_table_from])) {

6708 6709
    for (uint64_t i = pThreadInfo->start_table_from;
            i <= pThreadInfo->end_table_to; i++) {
6710 6711 6712 6713
      tsubSeq = i - pThreadInfo->start_table_from;
      if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
          continue;
      }
6714

6715 6716
      st = taosGetTimestampMs();
      performancePrint("st: %"PRIu64" et: %"PRIu64" st-et: %"PRIu64"\n", st, et, (st - et));
6717
      res = taos_consume(tsub[tsubSeq]);
6718 6719 6720
      et = taosGetTimestampMs();
      performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st));

6721 6722
      if (res) {
          if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6723
              sprintf(pThreadInfo->filePath, "%s-%d",
6724 6725
                      g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
6726
              fetchResult(res, pThreadInfo);
6727 6728
          }
          if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6729
              sprintf(pThreadInfo->filePath, "%s-%d",
6730 6731
                      g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
6732
              fetchResult(res, pThreadInfo);
6733 6734 6735
          }
          consumed[tsubSeq] ++;

6736
          if ((g_queryInfo.superQueryInfo.resubAfterConsume != -1)
6737
                  && (consumed[tsubSeq] >=
6738
                      g_queryInfo.superQueryInfo.resubAfterConsume)) {
6739 6740 6741 6742
              printf("keepProgress:%d, resub super table query: %"PRIu64"\n",
                      g_queryInfo.superQueryInfo.subscribeKeepProgress,
                      pThreadInfo->querySeq);
              taos_unsubscribe(tsub[tsubSeq],
6743
                    g_queryInfo.superQueryInfo.subscribeKeepProgress);
6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756
              consumed[tsubSeq]= 0;
              tsub[tsubSeq] = subscribeImpl(
                      STABLE_CLASS,
                      pThreadInfo, subSqlstr, topic,
                      g_queryInfo.superQueryInfo.subscribeRestart,
                      g_queryInfo.superQueryInfo.subscribeInterval
                      );
              if (NULL == tsub[tsubSeq]) {
                  taos_close(pThreadInfo->taos);
                  return NULL;
              }
          }
      }
H
hzcheng 已提交
6757 6758
    }
  }
6759
  taos_free_result(res);
6760

6761 6762
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6763 6764
    tsubSeq = i - pThreadInfo->start_table_from;
    taos_unsubscribe(tsub[tsubSeq], 0);
6765
  }
6766

6767
  taos_close(pThreadInfo->taos);
H
hzcheng 已提交
6768 6769 6770
  return NULL;
}

6771 6772
static void *specifiedSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6773
//  TAOS_SUB*  tsub = NULL;
6774

6775
  if (pThreadInfo->taos == NULL) {
6776
    pThreadInfo->taos = taos_connect(g_queryInfo.host,
6777 6778 6779 6780
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
6781
    if (pThreadInfo->taos == NULL) {
6782
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6783
            pThreadInfo->threadID, taos_errstr(NULL));
6784 6785 6786 6787
      return NULL;
    }
  }

6788 6789
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6790 6791
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6792 6793
    return NULL;
  }
6794

6795 6796 6797 6798
  sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
          "taosdemo-subscribe-%"PRIu64"-%d",
          pThreadInfo->querySeq,
          pThreadInfo->threadID);
6799
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') {
6800
      sprintf(pThreadInfo->filePath, "%s-%d",
6801
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
6802
                pThreadInfo->threadID);
6803
  }
6804 6805
  g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl(
          SPECIFIED_CLASS, pThreadInfo,
6806
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
6807
          g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
6808 6809
          g_queryInfo.specifiedQueryInfo.subscribeRestart,
          g_queryInfo.specifiedQueryInfo.subscribeInterval);
6810
  if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) {
6811 6812
      taos_close(pThreadInfo->taos);
      return NULL;
6813
  }
6814

6815
  // start loop to consume result
6816

6817
  g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0;
6818 6819 6820 6821
  while((g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq] == -1)
          || (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] <
              g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq])) {

6822
      if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) {
6823 6824
        continue;
      }
6825

6826 6827 6828
      g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] = taos_consume(
              g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]);
      if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) {
6829
          if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6830
              sprintf(pThreadInfo->filePath, "%s-%d",
6831 6832
                      g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
6833
              fetchResult(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], pThreadInfo);
6834 6835
          }

6836
          g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] ++;
6837
          if ((g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq] != -1)
6838
                && (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] >=
6839 6840 6841 6842
                    g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
              printf("keepProgress:%d, resub specified query: %"PRIu64"\n",
                    g_queryInfo.specifiedQueryInfo.subscribeKeepProgress,
                    pThreadInfo->querySeq);
6843 6844
              g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0;
              taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID],
6845
                      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
6846
              g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl(
6847 6848 6849
                      SPECIFIED_CLASS,
                      pThreadInfo,
                      g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
6850
                      g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
6851 6852
                      g_queryInfo.specifiedQueryInfo.subscribeRestart,
                      g_queryInfo.specifiedQueryInfo.subscribeInterval);
6853
              if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) {
6854 6855
                taos_close(pThreadInfo->taos);
                return NULL;
6856 6857
              }
          }
6858 6859
      }
  }
6860 6861
  taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]);
  taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->querySeq], 0);
6862
  taos_close(pThreadInfo->taos);
6863

H
hzcheng 已提交
6864 6865 6866
  return NULL;
}

6867
static int subscribeTestProcess() {
6868
  setupForAnsiEscape();
6869
  printfQueryMeta();
6870
  resetAfterAnsiEscape();
6871

6872
  prompt();
6873

6874 6875 6876 6877 6878
  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
6879
          g_queryInfo.port);
6880
  if (taos == NULL) {
6881 6882
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6883 6884 6885
    exit(-1);
  }

6886
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6887 6888
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6889 6890 6891
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6892 6893
  }

6894 6895
  taos_close(taos); // TODO: workaround to use separate taos connection;

6896 6897
  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
6898 6899 6900 6901 6902

  pthread_t  *pidsOfStable  = NULL;
  threadInfo *infosOfStable = NULL;

  //==== create threads for query for specified table
6903
  if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) {
6904
    debugPrint("%s() LN%d, sepcified query sqlCount %d.\n",
6905
              __func__, __LINE__,
6906 6907 6908
              g_queryInfo.specifiedQueryInfo.sqlCount);
  } else {
    if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) {
6909
        errorPrint("%s() LN%d, sepcified query sqlCount %d.\n",
6910 6911 6912 6913
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount);
        exit(-1);
    }
6914

6915
    pids  = malloc(
6916 6917 6918
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(pthread_t));
6919
    infos = malloc(
6920 6921 6922
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(threadInfo));
6923 6924 6925 6926
    if ((NULL == pids) || (NULL == infos)) {
        errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
        exit(-1);
    }
6927

6928 6929 6930
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
            uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
6931 6932 6933 6934 6935
            threadInfo *pThreadInfo = infos + seq;
            pThreadInfo->threadID = seq;
            pThreadInfo->querySeq = i;
            pThreadInfo->taos = NULL;  // TODO: workaround to use separate taos connection;
            pthread_create(pids + seq, NULL, specifiedSubscribe, pThreadInfo);
6936
        }
6937
    }
H
hzcheng 已提交
6938
  }
6939

6940
  //==== create threads for super table query
6941
  if (g_queryInfo.superQueryInfo.sqlCount <= 0) {
6942
    debugPrint("%s() LN%d, super table query sqlCount %d.\n",
6943
              __func__, __LINE__,
6944
              g_queryInfo.superQueryInfo.sqlCount);
6945 6946
  } else {
    if ((g_queryInfo.superQueryInfo.sqlCount > 0)
6947
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
6948 6949 6950
        pidsOfStable  = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
6951
            sizeof(pthread_t));
6952 6953 6954
        infosOfStable = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
6955
            sizeof(threadInfo));
6956 6957
        if ((NULL == pidsOfStable) || (NULL == infosOfStable)) {
            errorPrint("%s() LN%d, malloc failed for create threads\n",
6958
              __func__, __LINE__);
6959 6960 6961
            // taos_close(taos);
            exit(-1);
        }
6962

6963 6964
        int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
        int threads = g_queryInfo.superQueryInfo.threadCnt;
6965

6966 6967 6968 6969 6970
        int64_t a = ntables / threads;
        if (a < 1) {
            threads = ntables;
            a = 1;
        }
6971

6972 6973 6974 6975
        int64_t b = 0;
        if (threads != 0) {
            b = ntables % threads;
        }
6976

6977
        for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
6978
            uint64_t startFrom = 0;
6979 6980
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
6981 6982 6983 6984 6985 6986 6987 6988 6989
                threadInfo *pThreadInfo = infosOfStable + seq;
                pThreadInfo->threadID = seq;
                pThreadInfo->querySeq = i;

                pThreadInfo->start_table_from = startFrom;
                pThreadInfo->ntables = j<b?a+1:a;
                pThreadInfo->end_table_to = j<b?startFrom+a:startFrom+a-1;
                startFrom = pThreadInfo->end_table_to + 1;
                pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection;
6990
                pthread_create(pidsOfStable + seq,
6991
                        NULL, superSubscribe, pThreadInfo);
6992 6993
            }
        }
6994

6995
        g_queryInfo.superQueryInfo.threadCnt = threads;
6996

6997 6998 6999 7000 7001 7002
        for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                pthread_join(pidsOfStable[seq], NULL);
            }
        }
7003
    }
H
hzcheng 已提交
7004
  }
7005

7006 7007 7008 7009 7010
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
    for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
        uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
        pthread_join(pids[seq], NULL);
    }
7011
  }
H
hzcheng 已提交
7012

7013
  tmfree((char*)pids);
7014
  tmfree((char*)infos);
H
hzcheng 已提交
7015

7016 7017
  tmfree((char*)pidsOfStable);
  tmfree((char*)infosOfStable);
7018
//   taos_close(taos);
7019
  return 0;
H
hzcheng 已提交
7020 7021
}

7022
static void initOfInsertMeta() {
7023
  memset(&g_Dbs, 0, sizeof(SDbs));
7024

7025
  // set default values
7026
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7027
  g_Dbs.port = 6030;
7028 7029
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
7030
  g_Dbs.threadCount = 2;
7031 7032

  g_Dbs.use_metric = g_args.use_metric;
L
Liu Tao 已提交
7033 7034
}

7035
static void initOfQueryMeta() {
7036
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
7037

7038
  // set default values
7039
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7040
  g_queryInfo.port = 6030;
7041 7042
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
L
Liu Tao 已提交
7043 7044
}

7045
static void setParaFromArg(){
7046
  if (g_args.host) {
7047
    tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE);
7048
  } else {
7049
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7050
  }
L
Liu Tao 已提交
7051

7052
  if (g_args.user) {
7053
    tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE);
7054
  }
7055 7056

  if (g_args.password) {
7057
    tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE);
7058 7059
  }

7060 7061
  if (g_args.port) {
    g_Dbs.port = g_args.port;
7062
  }
L
Liu Tao 已提交
7063

7064 7065 7066
  g_Dbs.threadCount = g_args.num_of_threads;
  g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;

7067 7068
  g_Dbs.dbCount = 1;
  g_Dbs.db[0].drop = 1;
7069

7070
  tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE);
7071
  g_Dbs.db[0].dbCfg.replica = g_args.replica;
7072
  tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
7073

7074
  tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
7075 7076 7077 7078 7079

  g_Dbs.use_metric = g_args.use_metric;
  g_Dbs.insert_only = g_args.insert_only;

  g_Dbs.do_aggreFunc = true;
L
Liu Tao 已提交
7080

7081 7082
  char dataString[STRING_LEN];
  char **data_type = g_args.datatype;
7083

7084
  memset(dataString, 0, STRING_LEN);
L
Liu Tao 已提交
7085

7086 7087
  if (strcasecmp(data_type[0], "BINARY") == 0
          || strcasecmp(data_type[0], "BOOL") == 0
7088
          || strcasecmp(data_type[0], "NCHAR") == 0 ) {
7089
    g_Dbs.do_aggreFunc = false;
L
Liu Tao 已提交
7090
  }
H
hzcheng 已提交
7091

7092 7093 7094 7095 7096
  if (g_args.use_metric) {
    g_Dbs.db[0].superTblCount = 1;
    tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE);
    g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
    g_Dbs.threadCount = g_args.num_of_threads;
7097
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
7098
    g_Dbs.asyncMode = g_args.async_mode;
7099

7100 7101 7102 7103
    g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
    g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
    g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
7104
    tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix,
7105 7106 7107
            g_args.tb_prefix, MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE);
7108
    tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp,
7109
            "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
7110
    g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
7111

7112
    g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
7113
    g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len;
7114

7115 7116 7117 7118 7119
    g_Dbs.db[0].superTbls[0].columnCount = 0;
    for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
      if (data_type[i] == NULL) {
        break;
      }
7120

7121
      tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
7122
              data_type[i], MAX_TB_NAME_SIZE);
7123
      g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;
7124 7125
      g_Dbs.db[0].superTbls[0].columnCount++;
    }
7126

7127 7128 7129
    if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
      g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
    } else {
7130 7131 7132 7133
      for (int i = g_Dbs.db[0].superTbls[0].columnCount;
              i < g_args.num_of_CPR; i++) {
        tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
                "INT", MAX_TB_NAME_SIZE);
7134
        g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;
7135 7136 7137
        g_Dbs.db[0].superTbls[0].columnCount++;
      }
    }
7138

7139 7140
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType,
            "INT", MAX_TB_NAME_SIZE);
7141
    g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;
7142

7143 7144
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType,
            "BINARY", MAX_TB_NAME_SIZE);
7145 7146
    g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;
    g_Dbs.db[0].superTbls[0].tagCount = 2;
7147
  } else {
7148 7149
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
    g_Dbs.db[0].superTbls[0].tagCount = 0;
H
hzcheng 已提交
7150
  }
7151 7152 7153 7154 7155 7156
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
7157

7158 7159 7160
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
7161 7162 7163
    exit(-1);
  }

7164 7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

7188
static void querySqlFile(TAOS* taos, char* sqlFile)
7189 7190 7191 7192 7193 7194
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
7195

7196 7197 7198 7199 7200 7201
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

7202
  double t = taosGetTimestampMs();
7203

7204
  while((read_len = tgetline(&line, &line_len, fp)) != -1) {
7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
7217
    }
7218 7219

    memcpy(cmd + cmd_len, line, read_len);
7220 7221 7222
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) {
        errorPrint("%s() LN%d, queryDbExec %s failed!\n",
               __func__, __LINE__, cmd);
7223 7224 7225 7226 7227
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
7228 7229
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
7230 7231
  }

7232
  t = taosGetTimestampMs() - t;
7233
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
7234

7235 7236 7237 7238
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
7239 7240
}

7241
static void testMetaFile() {
7242
    if (INSERT_TEST == g_args.test_mode) {
7243 7244
      if (g_Dbs.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
7245

7246
      insertTestProcess();
7247

7248
    } else if (QUERY_TEST == g_args.test_mode) {
7249 7250
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7251

7252
      queryTestProcess();
7253

7254
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
7255 7256
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7257

7258
      subscribeTestProcess();
7259

7260 7261 7262
    }  else {
      ;
    }
7263
}
7264

7265
static void queryResult() {
7266
  // query data
7267

7268
  pthread_t read_id;
7269 7270 7271 7272
  threadInfo *pThreadInfo = malloc(sizeof(threadInfo));
  assert(pThreadInfo);
  pThreadInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
  pThreadInfo->start_table_from = 0;
7273

7274
  //pThreadInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
7275
  if (g_args.use_metric) {
7276 7277 7278 7279
    pThreadInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
    pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
    pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
    tstrncpy(pThreadInfo->tb_prefix,
7280 7281
          g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE);
  } else {
7282 7283 7284
    pThreadInfo->ntables = g_args.num_of_tables;
    pThreadInfo->end_table_to = g_args.num_of_tables -1;
    tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE);
7285 7286
  }

7287
  pThreadInfo->taos = taos_connect(
7288 7289 7290 7291 7292
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
          g_Dbs.port);
7293
  if (pThreadInfo->taos == NULL) {
7294 7295
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
7296
    free(pThreadInfo);
7297 7298 7299
    exit(-1);
  }

7300
  tstrncpy(pThreadInfo->filePath, g_Dbs.resultFile, MAX_FILE_NAME_LEN);
7301 7302

  if (!g_Dbs.use_metric) {
7303
    pthread_create(&read_id, NULL, readTable, pThreadInfo);
7304
  } else {
7305
    pthread_create(&read_id, NULL, readMetric, pThreadInfo);
7306 7307
  }
  pthread_join(read_id, NULL);
7308 7309
  taos_close(pThreadInfo->taos);
  free(pThreadInfo);
7310 7311
}

7312 7313
static void testCmdLine() {

7314 7315 7316 7317 7318 7319 7320 7321 7322 7323
  if (strlen(configDir)) {
    wordexp_t full_path;
    if (wordexp(configDir, &full_path, 0) != 0) {
      errorPrint( "Invalid path %s\n", configDir);
      return;
    }
    taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
    wordfree(&full_path);
  }

7324 7325
  g_args.test_mode = INSERT_TEST;
  insertTestProcess();
7326

7327
  if (false == g_Dbs.insert_only)
7328
    queryResult();
7329 7330
}

7331 7332 7333
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

7334
  debugPrint("meta file: %s\n", g_args.metaFile);
7335 7336 7337

  if (g_args.metaFile) {
    initOfInsertMeta();
7338
    initOfQueryMeta();
7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
7352 7353 7354 7355
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
7356
          g_Dbs.port);
7357
      querySqlFile(qtaos, g_args.sqlFile);
7358 7359 7360 7361 7362
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
7363 7364 7365

    if (g_dupstr)
        free(g_dupstr);
H
hzcheng 已提交
7366
  }
7367 7368

  return 0;
H
hzcheng 已提交
7369
}
7370