taosdemo.c 246.2 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

21
#include <stdint.h>
S
slguan 已提交
22
#define _GNU_SOURCE
23 24
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
41
#else
S
TD-1057  
Shengliang Guan 已提交
42 43
  #include <regex.h>
  #include <stdio.h>
44
#endif
H
hzcheng 已提交
45

46
#include <assert.h>
47
#include <stdlib.h>
48 49
#include "cJSON.h"

50
#include "os.h"
H
hzcheng 已提交
51
#include "taos.h"
52
#include "taoserror.h"
H
Hui Li 已提交
53
#include "tutil.h"
S
slguan 已提交
54

55 56 57
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
58 59
extern char configDir[];

60 61 62 63
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

64 65 66 67 68 69
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};
70

71 72
#define MAX_RECORDS_PER_REQ     32766

73 74
#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
75
#define COND_BUF_LEN        BUFFER_SIZE - 30
76 77
#define MAX_USERNAME_SIZE  64
#define MAX_PASSWORD_SIZE  64
78
#define MAX_DB_NAME_SIZE   64
79
#define MAX_HOSTNAME_SIZE  64
80
#define MAX_TB_NAME_SIZE   64
81
#define MAX_DATA_SIZE      (16*1024)+20     // max record len: 16*1024, timestamp string and ,('') need extra space
82 83 84 85 86 87 88 89 90 91
#define MAX_NUM_DATATYPE   10
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
#define MAX_FILE_NAME_LEN  256

#define   MAX_SAMPLES_ONCE_FROM_FILE   10000
#define   MAX_NUM_DATATYPE 10

#define   MAX_DB_COUNT           8
92
#define   MAX_SUPER_TABLE_COUNT  200
93 94 95
#define   MAX_COLUMN_COUNT       1024
#define   MAX_TAG_COUNT          128

96
#define   MAX_QUERY_SQL_COUNT    100
97
#define   MAX_QUERY_SQL_LENGTH   1024
98 99

#define   MAX_DATABASE_COUNT     256
100
#define INPUT_BUF_LEN   256
101

102
#define DEFAULT_TIMESTAMP_STEP  1
103

104

105 106 107 108 109
typedef enum CREATE_SUB_TALBE_MOD_EN {
  PRE_CREATE_SUBTBL,
  AUTO_CREATE_SUBTBL,
  NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
110

111 112
typedef enum TALBE_EXISTS_EN {
  TBL_NO_EXISTS,
113
  TBL_ALREADY_EXISTS,
114 115 116
  TBL_EXISTS_BUTT
} TALBE_EXISTS_EN;

117
enum enumSYNC_MODE {
118 119
  SYNC_MODE,
  ASYNC_MODE,
120 121
  MODE_BUT
};
122

123 124 125 126 127 128
typedef enum enumQUERY_CLASS {
    SPECIFIED_CLASS,
    STABLE_CLASS,
    CLASS_BUT
} QUERY_CLASS;

129 130 131 132 133 134
typedef enum enum_INSERT_MODE {
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
} INSERT_MODE;

135
typedef enum enumQUERY_TYPE {
136
  NO_INSERT_TYPE,
137
  INSERT_TYPE,
138
  QUERY_TYPE_BUT
139
} QUERY_TYPE;
140 141 142 143 144 145 146

enum _show_db_index {
  TSDB_SHOW_DB_NAME_INDEX,
  TSDB_SHOW_DB_CREATED_TIME_INDEX,
  TSDB_SHOW_DB_NTABLES_INDEX,
  TSDB_SHOW_DB_VGROUPS_INDEX,
  TSDB_SHOW_DB_REPLICA_INDEX,
147
  TSDB_SHOW_DB_QUORUM_INDEX,
148 149 150 151 152 153 154 155 156 157
  TSDB_SHOW_DB_DAYS_INDEX,
  TSDB_SHOW_DB_KEEP_INDEX,
  TSDB_SHOW_DB_CACHE_INDEX,
  TSDB_SHOW_DB_BLOCKS_INDEX,
  TSDB_SHOW_DB_MINROWS_INDEX,
  TSDB_SHOW_DB_MAXROWS_INDEX,
  TSDB_SHOW_DB_WALLEVEL_INDEX,
  TSDB_SHOW_DB_FSYNC_INDEX,
  TSDB_SHOW_DB_COMP_INDEX,
  TSDB_SHOW_DB_CACHELAST_INDEX,
158
  TSDB_SHOW_DB_PRECISION_INDEX,
159 160 161 162 163 164 165 166 167 168
  TSDB_SHOW_DB_UPDATE_INDEX,
  TSDB_SHOW_DB_STATUS_INDEX,
  TSDB_MAX_SHOW_DB
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
  TSDB_SHOW_STABLES_NAME_INDEX,
  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
  TSDB_SHOW_STABLES_COLUMNS_INDEX,
169 170
  TSDB_SHOW_STABLES_METRIC_INDEX,
  TSDB_SHOW_STABLES_UID_INDEX,
171
  TSDB_SHOW_STABLES_TID_INDEX,
172
  TSDB_SHOW_STABLES_VGID_INDEX,
173
  TSDB_MAX_SHOW_STABLES
174 175
};

176 177 178 179 180 181 182 183 184 185 186 187 188 189
enum _describe_table_index {
  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
  TSDB_MAX_DESCRIBE_METRIC
};

typedef struct {
  char field[TSDB_COL_NAME_LEN + 1];
  char type[16];
  int length;
  char note[128];
} SColDes;
H
hzcheng 已提交
190 191

/* Used by main to communicate with parse_opt. */
192 193
static char *g_dupstr = NULL;

194 195
typedef struct SArguments_S {
  char *   metaFile;
196
  uint32_t test_mode;
197 198 199 200 201 202 203 204 205
  char *   host;
  uint16_t port;
  char *   user;
  char *   password;
  char *   database;
  int      replica;
  char *   tb_prefix;
  char *   sqlFile;
  bool     use_metric;
206
  bool     drop_database;
207
  bool     insert_only;
208
  bool     answer_yes;
209
  bool     debug_print;
210
  bool     verbose_print;
211
  bool     performance_print;
212
  char *   output_file;
213
  bool     async_mode;
214
  char *   datatype[MAX_NUM_DATATYPE + 1];
215 216 217 218
  uint32_t len_of_binary;
  uint32_t num_of_CPR;
  uint32_t num_of_threads;
  uint64_t insert_interval;
219
  int64_t  query_times;
220 221 222
  uint64_t interlace_rows;
  uint64_t num_of_RPR;                  // num_of_records_per_req
  uint64_t max_sql_len;
223 224
  int64_t  num_of_tables;
  int64_t  num_of_DPT;
225
  int      abort;
226 227
  int      disorderRatio;               // 0: no disorder, >0: x%
  int      disorderRange;               // ms or us by database precision
228
  uint32_t method_of_delete;
229
  char **  arg_list;
230 231
  uint64_t totalInsertRows;
  uint64_t totalAffectedRows;
232 233 234
} SArguments;

typedef struct SColumn_S {
235 236 237
  char      field[TSDB_COL_NAME_LEN + 1];
  char      dataType[MAX_TB_NAME_SIZE];
  uint32_t  dataLen;
238
  char      note[128];
239 240 241
} StrColumn;

typedef struct SSuperTable_S {
242
  char         sTblName[MAX_TB_NAME_SIZE+1];
243
  int64_t      childTblCount;
244
  bool         childTblExists;          // 0: no, 1: yes
245 246
  uint64_t     batchCreateTableNum;     // 0: no batch,  > 0: batch table number in one sql
  uint8_t      autoCreateTable;         // 0: create sub table, 1: auto create sub table
247
  char         childTblPrefix[MAX_TB_NAME_SIZE];
248
  char         dataSource[MAX_TB_NAME_SIZE+1];  // rand_gen or sample
249
  char         insertMode[MAX_TB_NAME_SIZE];    // taosc, rest
250
  int64_t      childTblLimit;
251
  uint64_t     childTblOffset;
252

253
//  int          multiThreadWriteOneTbl;  // 0: no, 1: yes
254
  uint64_t     interlaceRows;           //
255 256
  int          disorderRatio;           // 0: no disorder, >0: x%
  int          disorderRange;           // ms or us by database precision
257
  uint64_t     maxSqlLen;               //
258

259
  uint64_t     insertInterval;          // insert interval, will override global insert interval
260
  int64_t      insertRows;
261
  int64_t      timeStampStep;
262
  char         startTimestamp[MAX_TB_NAME_SIZE];
263
  char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
264 265
  char         sampleFile[MAX_FILE_NAME_LEN+1];
  char         tagsFile[MAX_FILE_NAME_LEN+1];
266

267
  uint32_t     columnCount;
268
  StrColumn    columns[MAX_COLUMN_COUNT];
269
  uint32_t     tagCount;
270 271 272
  StrColumn    tags[MAX_TAG_COUNT];

  char*        childTblName;
273
  char*        colsOfCreateChildTable;
274 275
  uint64_t     lenOfOneRow;
  uint64_t     lenOfTagOfOneRow;
276 277 278 279 280

  char*        sampleDataBuf;
  //int          sampleRowCount;
  //int          sampleUsePos;

281
  uint32_t     tagSource;    // 0: rand, 1: tag sample
282
  char*        tagDataBuf;
283 284
  uint32_t     tagSampleCount;
  uint32_t     tagUsePos;
285

286
  // statistics
287 288
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
289 290 291 292 293
} SSuperTable;

typedef struct {
  char     name[TSDB_DB_NAME_LEN + 1];
  char     create_time[32];
294
  int64_t  ntables;
295
  int32_t  vgroups;
296 297
  int16_t  replica;
  int16_t  quorum;
298
  int16_t  days;
299 300 301 302 303 304 305 306 307 308 309 310 311 312
  char     keeplist[32];
  int32_t  cache; //MB
  int32_t  blocks;
  int32_t  minrows;
  int32_t  maxrows;
  int8_t   wallevel;
  int32_t  fsync;
  int8_t   comp;
  int8_t   cachelast;
  char     precision[8];   // time resolution
  int8_t   update;
  char     status[16];
} SDbInfo;

313
typedef struct SDbCfg_S {
314
//  int       maxtablesPerVnode;
315 316
  uint32_t  minRows;        // 0 means default
  uint32_t  maxRows;        // 0 means default
317 318
  int       comp;
  int       walLevel;
319
  int       cacheLast;
320
  int       fsync;
321 322 323 324 325 326 327
  int       replica;
  int       update;
  int       keep;
  int       days;
  int       cache;
  int       blocks;
  int       quorum;
328
  char      precision[MAX_TB_NAME_SIZE];
329 330 331 332
} SDbCfg;

typedef struct SDataBase_S {
  char         dbName[MAX_DB_NAME_SIZE];
333
  bool         drop;  // 0: use exists, 1: if exists, drop then new create
334
  SDbCfg       dbCfg;
335
  uint64_t     superTblCount;
336 337 338 339
  SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
} SDataBase;

typedef struct SDbs_S {
340
  char         cfgDir[MAX_FILE_NAME_LEN+1];
341
  char         host[MAX_HOSTNAME_SIZE];
342 343
  struct sockaddr_in serv_addr;

344
  uint16_t     port;
345 346
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
347
  char         resultFile[MAX_FILE_NAME_LEN+1];
348 349 350
  bool         use_metric;
  bool         insert_only;
  bool         do_aggreFunc;
351
  bool         asyncMode;
352

353 354 355
  uint32_t     threadCount;
  uint32_t     threadCountByCreateTbl;
  uint32_t     dbCount;
356 357 358
  SDataBase    db[MAX_DB_COUNT];

  // statistics
359 360
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
361

362 363
} SDbs;

364
typedef struct SpecifiedQueryInfo_S {
365
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
366
  uint32_t     concurrent;
367
  int          sqlCount;
368
  uint32_t     asyncMode; // 0: sync, 1: async
369 370
  uint64_t     subscribeInterval; // ms
  uint64_t     queryTimes;
371
  bool         subscribeRestart;
372
  int          subscribeKeepProgress;
373
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
374
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
375
  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
376
  int          endAfterConsume[MAX_QUERY_SQL_COUNT];
377
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
378 379 380
  char         topic[MAX_QUERY_SQL_COUNT][32];
  int          consumed[MAX_QUERY_SQL_COUNT];
  TAOS_RES*    res[MAX_QUERY_SQL_COUNT];
381
  uint64_t     totalQueried;
382
} SpecifiedQueryInfo;
383

384
typedef struct SuperQueryInfo_S {
385
  char         sTblName[MAX_TB_NAME_SIZE+1];
386 387
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint32_t     threadCnt;
388
  uint32_t     asyncMode; // 0: sync, 1: async
389
  uint64_t     subscribeInterval; // ms
390
  bool         subscribeRestart;
391
  int          subscribeKeepProgress;
392
  uint64_t     queryTimes;
393
  int64_t      childTblCount;
394
  char         childTblPrefix[MAX_TB_NAME_SIZE];
395
  int          sqlCount;
396
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
397
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
398
  int          resubAfterConsume;
399
  int          endAfterConsume;
400
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
401

402
  char*        childTblName;
403
  uint64_t     totalQueried;
404
} SuperQueryInfo;
405 406

typedef struct SQueryMetaInfo_S {
407
  char         cfgDir[MAX_FILE_NAME_LEN+1];
408
  char         host[MAX_HOSTNAME_SIZE];
409
  uint16_t     port;
410
  struct       sockaddr_in serv_addr;
411 412
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
413
  char         dbName[MAX_DB_NAME_SIZE+1];
414
  char         queryMode[MAX_TB_NAME_SIZE];  // taosc, rest
415

416
  SpecifiedQueryInfo  specifiedQueryInfo;
417
  SuperQueryInfo      superQueryInfo;
418
  uint64_t     totalQueried;
419 420 421
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
422 423 424 425
  TAOS *    taos;
  int       threadID;
  char      db_name[MAX_DB_NAME_SIZE+1];
  uint32_t  time_precision;
426 427
  char      filePath[4096];
  FILE      *fp;
428 429 430
  char      tb_prefix[MAX_TB_NAME_SIZE];
  uint64_t  start_table_from;
  uint64_t  end_table_to;
431
  int64_t   ntables;
432 433 434 435
  uint64_t  data_of_rate;
  int64_t   start_time;
  char*     cols;
  bool      use_metric;
436 437 438
  SSuperTable* superTblInfo;

  // for async insert
439 440
  tsem_t    lock_sem;
  int64_t   counter;
441 442
  uint64_t  st;
  uint64_t  et;
443
  uint64_t  lastTs;
444

445
  // sample data
446
  int64_t   samplePos;
447
  // statistics
448 449
  uint64_t  totalInsertRows;
  uint64_t  totalAffectedRows;
450 451

  // insert delay statistics
452 453 454 455 456
  uint64_t  cntDelay;
  uint64_t  totalDelay;
  uint64_t  avgDelay;
  uint64_t  maxDelay;
  uint64_t  minDelay;
457

458
  // seq of query or subscribe
459
  uint64_t  querySeq;   // sequence number of sql command
460
  TAOS_SUB*  tsub;
461

462 463
} threadInfo;

464
#ifdef WINDOWS
465 466
#define _CRT_RAND_S

467
#include <windows.h>
468 469 470 471 472
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
473 474
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
475 476
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
477 478 479 480

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

481
static void setupForAnsiEscape(void) {
482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
500
  }
501 502
}

503
static void resetAfterAnsiEscape(void) {
504
  // Reset colors
505
  printf("\x1b[0m");
506 507 508 509 510 511

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
512

513
static int taosRandom()
514 515 516 517 518 519
{
    int number;
    rand_s(&number);

    return number;
}
520
#else   // Not windows
521
static void setupForAnsiEscape(void) {}
522

523
static void resetAfterAnsiEscape(void) {
524 525 526
  // Reset colors
  printf("\x1b[0m");
}
527

528 529
#include <time.h>

530
static int taosRandom()
531
{
532
  return rand();
533 534
}

535
#endif // ifdef Windows
536

537
static void prompt();
538
static int createDatabasesAndStables();
539
static void createChildTables();
540
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet);
541
static int postProceSql(char *host, struct sockaddr_in *pServAddr,
542
        uint16_t port, char* sqlstr, threadInfo *pThreadInfo);
543 544
static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq,
                  int disorderRatio, int disorderRange);
545 546 547 548 549 550 551

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
552
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)",
553 554
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

555
SArguments g_args = {
556 557
                     NULL,            // metaFile
                     0,               // test_mode
558 559 560
                     "127.0.0.1",     // host
                     6030,            // port
                     "root",          // user
561
                     #ifdef _TD_POWER_
562 563 564 565 566 567 568 569
                     "powerdb",      // password
                     #else
                     "taosdata",      // password
                     #endif
                     "test",          // database
                     1,               // replica
                     "t",             // tb_prefix
                     NULL,            // sqlFile
570
                     true,            // use_metric
571
                     true,            // drop_database
572
                     true,            // insert_only
573
                     false,           // debug_print
574
                     false,           // verbose_print
575
                     false,           // performance statistic print
576 577 578 579
                     false,           // answer_yes;
                     "./output.txt",  // output_file
                     0,               // mode : sync or async
                     {
580 581 582 583
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
584 585
                     },
                     16,              // len_of_binary
586
                     4,               // num_of_CPR
587 588
                     10,              // num_of_connections/thread
                     0,               // insert_interval
589
                     1,               // query_times
590
                     0,               // interlace_rows;
591
                     30000,           // num_of_RPR
592
                     (1024*1024),         // max_sql_len
593 594 595 596 597 598 599 600 601 602 603 604
                     10000,           // num_of_tables
                     10000,           // num_of_DPT
                     0,               // abort
                     0,               // disorderRatio
                     1000,            // disorderRange
                     1,               // method_of_delete
                     NULL             // arg_list
};



static SDbs            g_Dbs;
605
static int64_t         g_totalChildTables = 0;
606 607 608 609
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

#define debugPrint(fmt, ...) \
610
    do { if (g_args.debug_print || g_args.verbose_print) \
611
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
612

613
#define verbosePrint(fmt, ...) \
614 615
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
616

617 618 619 620
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)

621 622 623 624
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)


625
///////////////////////////////////////////////////
H
hzcheng 已提交
626

627 628
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

629 630 631 632 633 634 635 636
#ifndef TAOSDEMO_COMMIT_SHA1
#define TAOSDEMO_COMMIT_SHA1 "unknown"
#endif

#ifndef TD_VERNUMBER
#define TD_VERNUMBER    "unknown"
#endif

637
#ifndef TAOSDEMO_STATUS
638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654
#define TAOSDEMO_STATUS "unknown"
#endif

static void printVersion() {
    char tdengine_ver[] = TD_VERNUMBER;
    char taosdemo_ver[] = TAOSDEMO_COMMIT_SHA1;
    char taosdemo_status[] = TAOSDEMO_STATUS;

    if (strlen(taosdemo_status) == 0) {
        printf("taosdemo verison %s-%s\n",
                tdengine_ver, taosdemo_ver);
    } else {
        printf("taosdemo verison %s-%s, status:%s\n",
                tdengine_ver, taosdemo_ver, taosdemo_status);
    }
}

655
static void printHelp() {
656
  char indent[10] = "        ";
657
  printf("%s%s%s%s\n", indent, "-f", indent,
658
          "The meta file to the execution procedure. Default is './meta.json'.");
659
  printf("%s%s%s%s\n", indent, "-u", indent,
660 661
          "The TDengine user name to use when connecting to the server. Default is 'root'.");
#ifdef _TD_POWER_
662
  printf("%s%s%s%s\n", indent, "-P", indent,
663
          "The password to use when connecting to the server. Default is 'powerdb'.");
664
  printf("%s%s%s%s\n", indent, "-c", indent,
665 666
          "Configuration directory. Default is '/etc/power/'.");
#else
667
  printf("%s%s%s%s\n", indent, "-P", indent,
668
          "The password to use when connecting to the server. Default is 'taosdata'.");
669
  printf("%s%s%s%s\n", indent, "-c", indent,
670
          "Configuration directory. Default is '/etc/taos/'.");
671
#endif
672
  printf("%s%s%s%s\n", indent, "-h", indent,
673
          "The host to connect to TDengine. Default is localhost.");
674
  printf("%s%s%s%s\n", indent, "-p", indent,
675
          "The TCP/IP port number to use for the connection. Default is 0.");
676
  printf("%s%s%s%s\n", indent, "-d", indent,
677
          "Destination database. Default is 'test'.");
678
  printf("%s%s%s%s\n", indent, "-a", indent,
679
          "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
680
  printf("%s%s%s%s\n", indent, "-m", indent,
681 682
          "Table prefix name. Default is 't'.");
  printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
683
  printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
684
  printf("%s%s%s%s\n", indent, "-o", indent,
685
          "Direct output to the named file. Default is './output.txt'.");
686
  printf("%s%s%s%s\n", indent, "-q", indent,
687
          "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC.");
688
  printf("%s%s%s%s\n", indent, "-b", indent,
689
          "The data_type of columns, default: INT,INT,INT,INT.");
690
  printf("%s%s%s%s\n", indent, "-w", indent,
691
          "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
692 693 694
  printf("%s%s%s%s%d\n", indent, "-l", indent,
          "The number of columns per record. Default is 4. Max values is ",
       MAX_NUM_DATATYPE);
695
  printf("%s%s%s%s\n", indent, "-T", indent,
696
          "The number of threads. Default is 10.");
697
  printf("%s%s%s%s\n", indent, "-i", indent,
698
          "The sleep time (ms) between insertion. Default is 0.");
699
  printf("%s%s%s%s\n", indent, "-r", indent,
700
          "The number of records per request. Default is 30000.");
701
  printf("%s%s%s%s\n", indent, "-t", indent,
702
          "The number of tables. Default is 10000.");
703
  printf("%s%s%s%s\n", indent, "-n", indent,
704 705 706
          "The number of records per table. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
  printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
707
  printf("%s%s%s%s\n", indent, "-O", indent,
708
          "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order.");
709
  printf("%s%s%s%s\n", indent, "-R", indent,
710
          "Out of order data's range, ms, default is 1000.");
711
  printf("%s%s%s%s\n", indent, "-g", indent,
712
          "Print debug info.");
713
  printf("%s%s%s\n", indent, "-V, --version\t",
714
          "Print version info.");
715 716
  printf("%s%s%s%s\n", indent, "--help\t", indent,
          "Print command line arguments list info.");
717
/*    printf("%s%s%s%s\n", indent, "-D", indent,
718 719 720 721
          "if elete database if exists. 0: no, 1: yes, default is 1");
          */
}

722 723 724 725 726 727 728 729 730 731 732 733 734 735 736
static bool isStringNumber(char *input)
{
  int len = strlen(input);
  if (0 == len) {
    return false;
  }

  for (int i = 0; i < len; i++) {
    if (!isdigit(input[i]))
      return false;
  }

  return true;
}

737
static void parse_args(int argc, char *argv[], SArguments *arguments) {
738 739 740 741 742

  for (int i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-f") == 0) {
      arguments->metaFile = argv[++i];
    } else if (strcmp(argv[i], "-c") == 0) {
743 744 745 746 747
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-c need a valid path following!\n");
        exit(EXIT_FAILURE);
      }
748
      tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN);
749
    } else if (strcmp(argv[i], "-h") == 0) {
750 751 752 753 754
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-h need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
755 756
      arguments->host = argv[++i];
    } else if (strcmp(argv[i], "-p") == 0) {
757 758 759 760 761 762
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-p need a number following!\n");
        exit(EXIT_FAILURE);
      }
763 764
      arguments->port = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-u") == 0) {
765 766 767 768 769
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-u need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
770 771
      arguments->user = argv[++i];
    } else if (strcmp(argv[i], "-P") == 0) {
772 773 774 775 776
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-P need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
777 778
      arguments->password = argv[++i];
    } else if (strcmp(argv[i], "-o") == 0) {
779 780 781 782 783
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-o need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
784 785
      arguments->output_file = argv[++i];
    } else if (strcmp(argv[i], "-s") == 0) {
786 787 788 789 790
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-s need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
791 792
      arguments->sqlFile = argv[++i];
    } else if (strcmp(argv[i], "-q") == 0) {
793 794
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
795
        printHelp();
796
        errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n");
797 798
        exit(EXIT_FAILURE);
      }
799
      arguments->async_mode = atoi(argv[++i]);
800
    } else if (strcmp(argv[i], "-T") == 0) {
801 802
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
803 804 805 806
        printHelp();
        errorPrint("%s", "\n\t-T need a number following!\n");
        exit(EXIT_FAILURE);
      }
807 808
      arguments->num_of_threads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
809 810
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
811 812 813 814
        printHelp();
        errorPrint("%s", "\n\t-i need a number following!\n");
        exit(EXIT_FAILURE);
      }
815
      arguments->insert_interval = atoi(argv[++i]);
816
    } else if (strcmp(argv[i], "-qt") == 0) {
817 818
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
819
        printHelp();
820
        errorPrint("%s", "\n\t-qt need a number following!\n");
821 822
        exit(EXIT_FAILURE);
      }
823
      arguments->query_times = atoi(argv[++i]);
824
    } else if (strcmp(argv[i], "-B") == 0) {
825 826
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
827 828 829 830
        printHelp();
        errorPrint("%s", "\n\t-B need a number following!\n");
        exit(EXIT_FAILURE);
      }
831
      arguments->interlace_rows = atoi(argv[++i]);
832
    } else if (strcmp(argv[i], "-r") == 0) {
833 834
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
835 836 837 838
        printHelp();
        errorPrint("%s", "\n\t-r need a number following!\n");
        exit(EXIT_FAILURE);
      }
839 840
      arguments->num_of_RPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-t") == 0) {
841 842 843 844 845 846
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-t need a number following!\n");
        exit(EXIT_FAILURE);
      }
847 848
      arguments->num_of_tables = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
849 850 851 852 853 854
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-n need a number following!\n");
        exit(EXIT_FAILURE);
      }
855 856
      arguments->num_of_DPT = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-d") == 0) {
857 858 859 860 861
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-d need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
862 863
      arguments->database = argv[++i];
    } else if (strcmp(argv[i], "-l") == 0) {
864 865 866 867 868 869
      if (argc == i+1) {
        if (!isStringNumber(argv[i+1])) {
            printHelp();
            errorPrint("%s", "\n\t-l need a number following!\n");
            exit(EXIT_FAILURE);
        }
870
      }
871
      arguments->num_of_CPR = atoi(argv[++i]);
872 873 874 875 876 877 878 879 880 881 882

      if (arguments->num_of_CPR > MAX_NUM_DATATYPE) {
          printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE);
          prompt();
          arguments->num_of_CPR = MAX_NUM_DATATYPE;
      }

      for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) {
          arguments->datatype[col] = NULL;
      }

883
    } else if (strcmp(argv[i], "-b") == 0) {
884 885 886 887 888
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-b need valid string following!\n");
        exit(EXIT_FAILURE);
      }
889 890
      ++i;
      if (strstr(argv[i], ",") == NULL) {
891
        // only one col
892 893 894 895 896 897 898
        if (strcasecmp(argv[i], "INT")
                && strcasecmp(argv[i], "FLOAT")
                && strcasecmp(argv[i], "TINYINT")
                && strcasecmp(argv[i], "BOOL")
                && strcasecmp(argv[i], "SMALLINT")
                && strcasecmp(argv[i], "BIGINT")
                && strcasecmp(argv[i], "DOUBLE")
899
                && strcasecmp(argv[i], "BINARY")
900 901
                && strcasecmp(argv[i], "NCHAR")) {
          printHelp();
902
          errorPrint("%s", "-b: Invalid data_type!\n");
903
          exit(EXIT_FAILURE);
904
        }
905
        arguments->datatype[0] = argv[i];
906
      } else {
907
        // more than one col
908
        int index = 0;
909 910
        g_dupstr = strdup(argv[i]);
        char *running = g_dupstr;
911
        char *token = strsep(&running, ",");
912
        while(token != NULL) {
913 914 915 916 917 918 919 920 921 922
          if (strcasecmp(token, "INT")
                  && strcasecmp(token, "FLOAT")
                  && strcasecmp(token, "TINYINT")
                  && strcasecmp(token, "BOOL")
                  && strcasecmp(token, "SMALLINT")
                  && strcasecmp(token, "BIGINT")
                  && strcasecmp(token, "DOUBLE")
                  && strcasecmp(token, "BINARY")
                  && strcasecmp(token, "NCHAR")) {
            printHelp();
923
            free(g_dupstr);
924
            errorPrint("%s", "-b: Invalid data_type!\n");
925
            exit(EXIT_FAILURE);
H
hzcheng 已提交
926
          }
927
          arguments->datatype[index++] = token;
928 929
          token = strsep(&running, ",");
          if (index >= MAX_NUM_DATATYPE) break;
S
TD-1057  
Shengliang Guan 已提交
930
        }
931
        arguments->datatype[index] = NULL;
932
      }
933
    } else if (strcmp(argv[i], "-w") == 0) {
934 935 936 937 938 939
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-w need a number following!\n");
        exit(EXIT_FAILURE);
      }
940 941
      arguments->len_of_binary = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-m") == 0) {
942 943 944 945 946 947
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-m need a number following!\n");
        exit(EXIT_FAILURE);
      }
948
      arguments->tb_prefix = argv[++i];
949 950
    } else if (strcmp(argv[i], "-N") == 0) {
      arguments->use_metric = false;
951
    } else if (strcmp(argv[i], "-x") == 0) {
952
      arguments->insert_only = false;
953
    } else if (strcmp(argv[i], "-y") == 0) {
954
      arguments->answer_yes = true;
955 956
    } else if (strcmp(argv[i], "-g") == 0) {
      arguments->debug_print = true;
957 958
    } else if (strcmp(argv[i], "-gg") == 0) {
      arguments->verbose_print = true;
959 960
    } else if (strcmp(argv[i], "-pp") == 0) {
      arguments->performance_print = true;
961
    } else if (strcmp(argv[i], "-O") == 0) {
962 963 964 965 966 967
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-O need a number following!\n");
        exit(EXIT_FAILURE);
      }
968

969
      arguments->disorderRatio = atoi(argv[++i]);
970

971
      if (arguments->disorderRatio > 50) {
972
        arguments->disorderRatio = 50;
973
      }
974

975
      if (arguments->disorderRatio < 0) {
976
        arguments->disorderRatio = 0;
977
      }
978

979
    } else if (strcmp(argv[i], "-R") == 0) {
980 981 982 983 984 985
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-R need a number following!\n");
        exit(EXIT_FAILURE);
      }
986

987
      arguments->disorderRange = atoi(argv[++i]);
988 989 990
      if (arguments->disorderRange < 0)
        arguments->disorderRange = 1000;

991
    } else if (strcmp(argv[i], "-a") == 0) {
992 993 994 995 996 997
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-a need a number following!\n");
        exit(EXIT_FAILURE);
      }
998 999 1000
      arguments->replica = atoi(argv[++i]);
      if (arguments->replica > 3 || arguments->replica < 1) {
          arguments->replica = 1;
1001
      }
1002 1003
    } else if (strcmp(argv[i], "-D") == 0) {
      arguments->method_of_delete = atoi(argv[++i]);
1004 1005 1006
      if (arguments->method_of_delete > 3) {
        errorPrint("%s", "\n\t-D need a valud (0~3) number following!\n");
        exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1007
      }
1008 1009 1010 1011
    } else if ((strcmp(argv[i], "--version") == 0) ||
        (strcmp(argv[i], "-V") == 0)){
      printVersion();
      exit(0);
1012 1013 1014 1015 1016
    } else if (strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else {
      printHelp();
1017
      errorPrint("%s", "ERROR: wrong options\n");
1018
      exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1019 1020
    }
  }
1021

1022 1023
  if (((arguments->debug_print) && (arguments->metaFile == NULL))
          || arguments->verbose_print) {
1024
    printf("###################################################################\n");
1025
    printf("# meta file:                         %s\n", arguments->metaFile);
1026
    printf("# Server IP:                         %s:%hu\n",
1027 1028 1029 1030 1031
            arguments->host == NULL ? "localhost" : arguments->host,
            arguments->port );
    printf("# User:                              %s\n", arguments->user);
    printf("# Password:                          %s\n", arguments->password);
    printf("# Use metric:                        %s\n", arguments->use_metric ? "true" : "false");
1032 1033 1034 1035 1036 1037 1038 1039 1040
    if (*(arguments->datatype)) {
        printf("# Specified data type:               ");
        for (int i = 0; i < MAX_NUM_DATATYPE; i++)
            if (arguments->datatype[i])
               printf("%s,", arguments->datatype[i]);
            else
                break;
        printf("\n");
    }
1041
    printf("# Insertion interval:                %"PRIu64"\n",
1042
            arguments->insert_interval);
1043
    printf("# Number of records per req:         %"PRIu64"\n",
1044
            arguments->num_of_RPR);
1045
    printf("# Max SQL length:                    %"PRIu64"\n",
1046
            arguments->max_sql_len);
1047
    printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
1048
    printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
1049
    printf("# Number of Tables:                  %"PRId64"\n",
1050
            arguments->num_of_tables);
1051
    printf("# Number of Data per Table:          %"PRId64"\n",
1052
            arguments->num_of_DPT);
1053 1054 1055 1056 1057
    printf("# Database name:                     %s\n", arguments->database);
    printf("# Table prefix:                      %s\n", arguments->tb_prefix);
    if (arguments->disorderRatio) {
      printf("# Data order:                        %d\n", arguments->disorderRatio);
      printf("# Data out of order rate:            %d\n", arguments->disorderRange);
1058

1059 1060 1061 1062
    }
    printf("# Delete method:                     %d\n", arguments->method_of_delete);
    printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
    printf("# Print debug info:                  %d\n", arguments->debug_print);
1063
    printf("# Print verbose info:                %d\n", arguments->verbose_print);
1064
    printf("###################################################################\n");
1065 1066

    prompt();
1067
  }
1068
}
H
hzcheng 已提交
1069

1070 1071 1072 1073
static bool getInfoFromJsonFile(char* file);
//static int generateOneRowDataForStb(SSuperTable* stbInfo);
//static int getDataIntoMemForStb(SSuperTable* stbInfo);
static void init_rand_data();
1074
static void tmfclose(FILE *fp) {
1075 1076 1077 1078
  if (NULL != fp) {
    fclose(fp);
  }
}
1079

1080
static void tmfree(char *buf) {
1081 1082
  if (NULL != buf) {
    free(buf);
H
Hui Li 已提交
1083
  }
1084
}
H
Hui Li 已提交
1085

1086
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
1087 1088 1089
  int i;
  TAOS_RES *res = NULL;
  int32_t   code = -1;
1090

1091 1092 1093 1094
  for (i = 0; i < 5; i++) {
    if (NULL != res) {
      taos_free_result(res);
      res = NULL;
H
hzcheng 已提交
1095
    }
1096

1097 1098 1099 1100
    res = taos_query(taos, command);
    code = taos_errno(res);
    if (0 == code) {
      break;
1101
    }
H
hzcheng 已提交
1102 1103
  }

1104
  verbosePrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
1105
  if (code != 0) {
1106
    if (!quiet) {
1107
      errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res));
1108
    }
1109 1110 1111
    taos_free_result(res);
    //taos_close(taos);
    return -1;
1112
  }
H
hzcheng 已提交
1113

1114 1115 1116 1117
  if (INSERT_TYPE == type) {
    int affectedRows = taos_affected_rows(res);
    taos_free_result(res);
    return affectedRows;
H
hzcheng 已提交
1118
  }
1119

1120
  taos_free_result(res);
1121 1122
  return 0;
}
H
hzcheng 已提交
1123

1124
static void appendResultBufToFile(char *resultBuf, threadInfo *pThreadInfo)
1125
{
1126 1127
  pThreadInfo->fp = fopen(pThreadInfo->filePath, "at");
  if (pThreadInfo->fp == NULL) {
1128 1129
      errorPrint(
              "%s() LN%d, failed to open result file: %s, result will not save to file\n",
1130
              __func__, __LINE__, pThreadInfo->filePath);
1131 1132
      return;
  }
1133

1134 1135 1136
  fprintf(pThreadInfo->fp, "%s", resultBuf);
  tmfclose(pThreadInfo->fp);
  pThreadInfo->fp = NULL;
1137
}
1138

1139
static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) {
1140 1141 1142 1143 1144
  TAOS_ROW    row = NULL;
  int         num_rows = 0;
  int         num_fields = taos_field_count(res);
  TAOS_FIELD *fields     = taos_fetch_fields(res);

1145 1146
  char* databuf = (char*) calloc(1, 100*1024*1024);
  if (databuf == NULL) {
1147 1148
    errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n",
            __func__, __LINE__);
1149 1150
    return ;
  }
H
hzcheng 已提交
1151

1152 1153
  int   totalLen = 0;
  char  temp[16000];
H
hzcheng 已提交
1154

1155
  // fetch the records row by row
1156
  while((row = taos_fetch_row(res))) {
1157
    if ((strlen(pThreadInfo->filePath) > 0)
1158
            && (totalLen >= 100*1024*1024 - 32000)) {
1159
        appendResultBufToFile(databuf, pThreadInfo);
1160 1161
        totalLen = 0;
        memset(databuf, 0, 100*1024*1024);
H
hzcheng 已提交
1162
    }
1163 1164 1165 1166 1167 1168
    num_rows++;
    int len = taos_print_row(temp, row, fields, num_fields);
    len += sprintf(temp + len, "\n");
    //printf("query result:%s\n", temp);
    memcpy(databuf + totalLen, temp, len);
    totalLen += len;
H
hzcheng 已提交
1169 1170
  }

1171
  verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n",
1172 1173 1174
          __func__, __LINE__, databuf, pThreadInfo->filePath);
  if (strlen(pThreadInfo->filePath) > 0) {
    appendResultBufToFile(databuf, pThreadInfo);
1175
  }
1176 1177
  free(databuf);
}
H
hzcheng 已提交
1178

1179
static void selectAndGetResult(
1180
        threadInfo *pThreadInfo, char *command)
1181 1182 1183 1184 1185 1186 1187 1188 1189 1190
{
  if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) {
    TAOS_RES *res = taos_query(pThreadInfo->taos, command);
    if (res == NULL || taos_errno(res) != 0) {
        errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n",
            __func__, __LINE__, command, taos_errstr(res));
        taos_free_result(res);
        return;
    }

1191
    fetchResult(res, pThreadInfo);
1192
    taos_free_result(res);
1193

1194 1195 1196 1197
  } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
      int retCode = postProceSql(
              g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port,
              command,
1198
              pThreadInfo);
1199 1200 1201 1202 1203 1204 1205 1206
      if (0 != retCode) {
        printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID);
      }

  } else {
      errorPrint("%s() LN%d, unknown query mode: %s\n",
        __func__, __LINE__, g_queryInfo.queryMode);
  }
1207
}
H
hzcheng 已提交
1208

1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247
static int32_t rand_bool(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 2;
}

static int32_t rand_tinyint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 128;
}

static int32_t rand_smallint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 32767;
}

static int32_t rand_int(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor];
}

static int64_t rand_bigint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randbigint[cursor];
}

static float rand_float(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
1248
  return randfloat[cursor];
1249 1250
}

1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267
#if 0
static const char charNum[] = "0123456789";

static void nonrand_string(char *, int) __attribute__ ((unused));   // reserve for debugging purpose
static void nonrand_string(char *str, int size)
{
  str[0] = 0;
  if (size > 0) {
    int n;
    for (n = 0; n < size; n++) {
      str[n] = charNum[n % 10];
    }
    str[n] = 0;
  }
}
#endif

1268
static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1269

1270
static void rand_string(char *str, int size) {
1271 1272 1273 1274
  str[0] = 0;
  if (size > 0) {
    //--size;
    int n;
1275
    for (n = 0; n < size; n++) {
1276
      int key = abs(rand_tinyint()) % (int)(sizeof(charset) - 1);
1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
      str[n] = charset[key];
    }
    str[n] = 0;
  }
}

static double rand_double() {
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randdouble[cursor];

}

static void init_rand_data() {
  for (int i = 0; i < MAX_PREPARED_RAND; i++){
1293 1294 1295 1296
    randint[i] = (int)(taosRandom() % 65535);
    randbigint[i] = (int64_t)(taosRandom() % 2147483648);
    randfloat[i] = (float)(taosRandom() / 1000.0);
    randdouble[i] = (double)(taosRandom() / 1000000.0);
1297 1298 1299
  }
}

1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1320
static int printfInsertMeta() {
1321 1322
    SHOW_PARSE_RESULT_START();

1323 1324
  printf("host:                       \033[33m%s:%u\033[0m\n",
          g_Dbs.host, g_Dbs.port);
1325 1326
  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
1327
  printf("configDir:                  \033[33m%s\033[0m\n", configDir);
1328 1329
  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
1330 1331
  printf("thread num of create table: \033[33m%d\033[0m\n",
          g_Dbs.threadCountByCreateTbl);
1332
  printf("top insert interval:        \033[33m%"PRIu64"\033[0m\n",
1333
          g_args.insert_interval);
1334
  printf("number of records per req:  \033[33m%"PRIu64"\033[0m\n",
1335
          g_args.num_of_RPR);
1336
  printf("max sql length:             \033[33m%"PRIu64"\033[0m\n",
1337
          g_args.max_sql_len);
1338 1339

  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
1340

1341 1342
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    printf("database[\033[33m%d\033[0m]:\n", i);
1343 1344
    printf("  database[%d] name:      \033[33m%s\033[0m\n",
            i, g_Dbs.db[i].dbName);
1345
    if (0 == g_Dbs.db[i].drop) {
1346 1347 1348
      printf("  drop:                  \033[33mno\033[0m\n");
    } else {
      printf("  drop:                  \033[33myes\033[0m\n");
1349 1350 1351
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
1352 1353
      printf("  blocks:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.blocks);
1354 1355
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
1356 1357
      printf("  cache:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.cache);
1358 1359
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
1360 1361
      printf("  days:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.days);
1362 1363
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
1364 1365
      printf("  keep:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.keep);
1366 1367
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
1368 1369
      printf("  replica:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.replica);
1370 1371
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
1372 1373
      printf("  update:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.update);
1374 1375
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
1376 1377
      printf("  minRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.minRows);
1378 1379
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
1380 1381
      printf("  maxRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.maxRows);
1382 1383 1384 1385 1386
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
1387 1388
      printf("  walLevel:              \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.walLevel);
1389 1390
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
1391 1392
      printf("  fsync:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.fsync);
1393 1394
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
1395 1396
      printf("  quorum:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.quorum);
1397 1398
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1399 1400
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1401 1402
        printf("  precision:             \033[33m%s\033[0m\n",
            g_Dbs.db[i].dbCfg.precision);
1403
      } else {
1404 1405 1406
        printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
                g_Dbs.db[i].dbCfg.precision);
        return -1;
1407 1408 1409
      }
    }

1410
    printf("  super table count:     \033[33m%"PRIu64"\033[0m\n",
1411
        g_Dbs.db[i].superTblCount);
1412 1413
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      printf("  super table[\033[33m%"PRIu64"\033[0m]:\n", j);
1414

1415 1416
      printf("      stbName:           \033[33m%s\033[0m\n",
          g_Dbs.db[i].superTbls[j].sTblName);
1417 1418 1419 1420 1421 1422 1423 1424

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
      }
1425

1426 1427 1428 1429 1430 1431 1432
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
      }
1433

1434
      printf("      childTblCount:     \033[33m%"PRId64"\033[0m\n",
1435 1436 1437 1438 1439 1440 1441
              g_Dbs.db[i].superTbls[j].childTblCount);
      printf("      childTblPrefix:    \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      printf("      dataSource:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      printf("      insertMode:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].insertMode);
1442
      if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
1443
        printf("      childTblLimit:     \033[33m%"PRId64"\033[0m\n",
1444
                g_Dbs.db[i].superTbls[j].childTblLimit);
1445
      }
1446
      if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) {
1447
        printf("      childTblOffset:    \033[33m%"PRIu64"\033[0m\n",
1448
                g_Dbs.db[i].superTbls[j].childTblOffset);
1449
      }
1450
      printf("      insertRows:        \033[33m%"PRId64"\033[0m\n",
1451
              g_Dbs.db[i].superTbls[j].insertRows);
1452
/*
1453
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1454
        printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
1455
      }else {
1456
        printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
1457
      }
1458
      */
1459
      printf("      interlaceRows:     \033[33m%"PRIu64"\033[0m\n",
1460
              g_Dbs.db[i].superTbls[j].interlaceRows);
1461 1462

      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1463
        printf("      stable insert interval:   \033[33m%"PRIu64"\033[0m\n",
1464 1465 1466
            g_Dbs.db[i].superTbls[j].insertInterval);
      }

1467 1468 1469 1470
      printf("      disorderRange:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      printf("      disorderRatio:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
1471
      printf("      maxSqlLen:         \033[33m%"PRIu64"\033[0m\n",
1472
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1473
      printf("      timeStampStep:     \033[33m%"PRId64"\033[0m\n",
1474 1475 1476 1477 1478 1479 1480 1481 1482
              g_Dbs.db[i].superTbls[j].timeStampStep);
      printf("      startTimestamp:    \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
      printf("      sampleFormat:      \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      printf("      sampleFile:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      printf("      tagsFile:          \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
1483
      printf("      columnCount:       \033[33m%d\033[0m\n",
1484
              g_Dbs.db[i].superTbls[j].columnCount);
1485 1486
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1487 1488 1489 1490
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "nchar", 5))) {
1491
          printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
1492 1493
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1494
        } else {
1495 1496
          printf("column[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType);
1497 1498 1499
        }
      }
      printf("\n");
1500 1501 1502

      printf("      tagCount:            \033[33m%d\033[0m\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1503 1504
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1505 1506 1507 1508
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1509
          printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
1510 1511
                  g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1512
        } else {
1513 1514
          printf("tag[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].tags[k].dataType);
1515
        }
1516 1517 1518 1519 1520
      }
      printf("\n");
    }
    printf("\n");
  }
1521 1522

  SHOW_PARSE_RESULT_END();
1523 1524

  return 0;
1525 1526 1527
}

static void printfInsertMetaToFile(FILE* fp) {
1528 1529

  SHOW_PARSE_RESULT_START_TO_FILE(fp);
1530

1531 1532
  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
  fprintf(fp, "user:                       %s\n", g_Dbs.user);
1533
  fprintf(fp, "configDir:                  %s\n", configDir);
1534 1535 1536
  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
1537 1538
  fprintf(fp, "number of records per req:  %"PRIu64"\n", g_args.num_of_RPR);
  fprintf(fp, "max sql length:             %"PRIu64"\n", g_args.max_sql_len);
1539
  fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
1540

1541 1542
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    fprintf(fp, "database[%d]:\n", i);
1543
    fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
1544
    if (0 == g_Dbs.db[i].drop) {
1545
      fprintf(fp, "  drop:                  no\n");
1546
    }else {
1547
      fprintf(fp, "  drop:                  yes\n");
1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1587 1588
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1589 1590
        fprintf(fp, "  precision:             %s\n",
                g_Dbs.db[i].dbCfg.precision);
1591
      } else {
1592 1593
        fprintf(fp, "  precision error:       %s\n",
                g_Dbs.db[i].dbCfg.precision);
1594 1595 1596
      }
    }

1597 1598
    fprintf(fp, "  super table count:     %"PRIu64"\n",
            g_Dbs.db[i].superTblCount);
1599 1600
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      fprintf(fp, "  super table[%d]:\n", j);
1601

1602 1603
      fprintf(fp, "      stbName:           %s\n",
              g_Dbs.db[i].superTbls[j].sTblName);
1604 1605 1606

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "no");
1607 1608
      } else if (AUTO_CREATE_SUBTBL
              == g_Dbs.db[i].superTbls[j].autoCreateTable) {
1609 1610 1611 1612
        fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
      } else {
        fprintf(fp, "      autoCreateTable:   %s\n",  "error");
      }
1613

1614 1615
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "no");
1616 1617
      } else if (TBL_ALREADY_EXISTS
              == g_Dbs.db[i].superTbls[j].childTblExists) {
1618 1619 1620 1621
        fprintf(fp, "      childTblExists:    %s\n",  "yes");
      } else {
        fprintf(fp, "      childTblExists:    %s\n",  "error");
      }
1622

1623
      fprintf(fp, "      childTblCount:     %"PRId64"\n",
1624 1625 1626 1627 1628 1629 1630
              g_Dbs.db[i].superTbls[j].childTblCount);
      fprintf(fp, "      childTblPrefix:    %s\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      fprintf(fp, "      dataSource:        %s\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      fprintf(fp, "      insertMode:        %s\n",
              g_Dbs.db[i].superTbls[j].insertMode);
1631
      fprintf(fp, "      insertRows:        %"PRId64"\n",
1632
              g_Dbs.db[i].superTbls[j].insertRows);
1633
      fprintf(fp, "      interlace rows:    %"PRIu64"\n",
1634
              g_Dbs.db[i].superTbls[j].interlaceRows);
1635
      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1636
        fprintf(fp, "      stable insert interval:   %"PRIu64"\n",
1637
                g_Dbs.db[i].superTbls[j].insertInterval);
1638
      }
1639
/*
1640
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1641
        fprintf(fp, "      multiThreadWriteOneTbl:  no\n");
1642
      }else {
1643
        fprintf(fp, "      multiThreadWriteOneTbl:  yes\n");
1644
      }
1645
      */
1646
      fprintf(fp, "      interlaceRows:     %"PRIu64"\n",
1647
              g_Dbs.db[i].superTbls[j].interlaceRows);
1648 1649 1650 1651
      fprintf(fp, "      disorderRange:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      fprintf(fp, "      disorderRatio:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
1652
      fprintf(fp, "      maxSqlLen:         %"PRIu64"\n",
1653
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1654

1655 1656 1657 1658
      fprintf(fp, "      timeStampStep:     %"PRId64"\n",
              g_Dbs.db[i].superTbls[j].timeStampStep);
      fprintf(fp, "      startTimestamp:    %s\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
1659 1660 1661 1662 1663 1664
      fprintf(fp, "      sampleFormat:      %s\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      fprintf(fp, "      sampleFile:        %s\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      fprintf(fp, "      tagsFile:          %s\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
1665

1666 1667
      fprintf(fp, "      columnCount:       %d\n        ",
              g_Dbs.db[i].superTbls[j].columnCount);
1668 1669
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1670 1671 1672
        if ((0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "binary", strlen("binary")))
1673 1674
                || (0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
1675 1676 1677 1678
                        "nchar", strlen("nchar")))) {
          fprintf(fp, "column[%d]:%s(%d) ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1679
        } else {
1680 1681
          fprintf(fp, "column[%d]:%s ",
                  k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
1682 1683 1684
        }
      }
      fprintf(fp, "\n");
1685 1686 1687

      fprintf(fp, "      tagCount:            %d\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1688 1689
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1690 1691 1692 1693
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1694 1695
          fprintf(fp, "tag[%d]:%s(%d) ",
                  k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
1696
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1697 1698
        } else {
          fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
1699
        }
1700 1701 1702 1703 1704
      }
      fprintf(fp, "\n");
    }
    fprintf(fp, "\n");
  }
1705

1706
  SHOW_PARSE_RESULT_END_TO_FILE(fp);
1707 1708 1709
}

static void printfQueryMeta() {
1710

1711
  SHOW_PARSE_RESULT_START();
1712

1713 1714
  printf("host:                    \033[33m%s:%u\033[0m\n",
          g_queryInfo.host, g_queryInfo.port);
1715 1716 1717 1718
  printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
  printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

  printf("\n");
1719 1720 1721

  if ((SUBSCRIBE_TEST == g_args.test_mode) || (QUERY_TEST == g_args.test_mode)) {
    printf("specified table query info:                   \n");
1722
    printf("sqlCount:       \033[33m%d\033[0m\n",
1723
      g_queryInfo.specifiedQueryInfo.sqlCount);
1724 1725 1726
    if (g_queryInfo.specifiedQueryInfo.sqlCount > 0) {
      printf("specified tbl query times:\n");
      printf("                \033[33m%"PRIu64"\033[0m\n",
1727
      g_queryInfo.specifiedQueryInfo.queryTimes);
1728 1729 1730
      printf("query interval: \033[33m%"PRIu64" ms\033[0m\n",
        g_queryInfo.specifiedQueryInfo.queryInterval);
      printf("top query times:\033[33m%"PRIu64"\033[0m\n", g_args.query_times);
1731
      printf("concurrent:     \033[33m%d\033[0m\n",
1732 1733
      g_queryInfo.specifiedQueryInfo.concurrent);
      printf("mod:            \033[33m%s\033[0m\n",
1734
        (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync");
1735
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1736
        g_queryInfo.specifiedQueryInfo.subscribeInterval);
1737
      printf("restart:        \033[33m%d\033[0m\n",
1738
        g_queryInfo.specifiedQueryInfo.subscribeRestart);
1739
      printf("keepProgress:   \033[33m%d\033[0m\n",
1740
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
1741

1742 1743
      for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        printf("  sql[%d]: \033[33m%s\033[0m\n",
1744 1745 1746 1747 1748 1749
            i, g_queryInfo.specifiedQueryInfo.sql[i]);
      }
      printf("\n");
    }

    printf("super table query info:\n");
1750
    printf("sqlCount:       \033[33m%d\033[0m\n",
1751 1752 1753 1754 1755 1756 1757
      g_queryInfo.superQueryInfo.sqlCount);

    if (g_queryInfo.superQueryInfo.sqlCount > 0) {
      printf("query interval: \033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryInterval);
      printf("threadCnt:      \033[33m%d\033[0m\n",
        g_queryInfo.superQueryInfo.threadCnt);
1758
      printf("childTblCount:  \033[33m%"PRId64"\033[0m\n",
1759 1760 1761 1762 1763 1764 1765
        g_queryInfo.superQueryInfo.childTblCount);
      printf("stable name:    \033[33m%s\033[0m\n",
        g_queryInfo.superQueryInfo.sTblName);
      printf("stb query times:\033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryTimes);

      printf("mod:            \033[33m%s\033[0m\n",
1766
        (g_queryInfo.superQueryInfo.asyncMode)?"async":"sync");
1767
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1768
        g_queryInfo.superQueryInfo.subscribeInterval);
1769
      printf("restart:        \033[33m%d\033[0m\n",
1770
        g_queryInfo.superQueryInfo.subscribeRestart);
1771
      printf("keepProgress:   \033[33m%d\033[0m\n",
1772 1773
        g_queryInfo.superQueryInfo.subscribeKeepProgress);

1774 1775 1776 1777 1778 1779
      for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
        printf("  sql[%d]: \033[33m%s\033[0m\n",
            i, g_queryInfo.superQueryInfo.sql[i]);
      }
      printf("\n");
    }
1780
  }
1781

1782
  SHOW_PARSE_RESULT_END();
1783 1784
}

1785
static char* formatTimestamp(char* buf, int64_t val, int precision) {
1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816
  time_t tt;
  if (precision == TSDB_TIME_PRECISION_MICRO) {
    tt = (time_t)(val / 1000000);
  } else {
    tt = (time_t)(val / 1000);
  }

/* comment out as it make testcases like select_with_tags.sim fail.
  but in windows, this may cause the call to localtime crash if tt < 0,
  need to find a better solution.
  if (tt < 0) {
    tt = 0;
  }
  */

#ifdef WINDOWS
  if (tt < 0) tt = 0;
#endif

  struct tm* ptm = localtime(&tt);
  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);

  if (precision == TSDB_TIME_PRECISION_MICRO) {
    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
  } else {
    sprintf(buf + pos, ".%03d", (int)(val % 1000));
  }

  return buf;
}

1817 1818 1819
static void xDumpFieldToFile(FILE* fp, const char* val,
        TAOS_FIELD* field, int32_t length, int precision) {

1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854
  if (val == NULL) {
    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
    return;
  }

  char buf[TSDB_MAX_BYTES_PER_ROW];
  switch (field->type) {
    case TSDB_DATA_TYPE_BOOL:
      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
      break;
    case TSDB_DATA_TYPE_TINYINT:
      fprintf(fp, "%d", *((int8_t *)val));
      break;
    case TSDB_DATA_TYPE_SMALLINT:
      fprintf(fp, "%d", *((int16_t *)val));
      break;
    case TSDB_DATA_TYPE_INT:
      fprintf(fp, "%d", *((int32_t *)val));
      break;
    case TSDB_DATA_TYPE_BIGINT:
      fprintf(fp, "%" PRId64, *((int64_t *)val));
      break;
    case TSDB_DATA_TYPE_FLOAT:
      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
      break;
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      memcpy(buf, val, length);
      buf[length] = 0;
      fprintf(fp, "\'%s\'", buf);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
1855
      formatTimestamp(buf, *(int64_t*)val, precision);
1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870
      fprintf(fp, "'%s'", buf);
      break;
    default:
      break;
  }
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  FILE* fp = fopen(fname, "at");
  if (fp == NULL) {
1871
    errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885
    return -1;
  }

  int num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int precision = taos_result_precision(tres);

  for (int col = 0; col < num_fields; col++) {
    if (col > 0) {
      fprintf(fp, ",");
    }
    fprintf(fp, "%s", fields[col].name);
  }
  fputc('\n', fp);
1886

1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906
  int numOfRows = 0;
  do {
    int32_t* length = taos_fetch_lengths(tres);
    for (int i = 0; i < num_fields; i++) {
      if (i > 0) {
        fputc(',', fp);
      }
      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
    }
    fputc('\n', fp);

    numOfRows++;
    row = taos_fetch_row(tres);
  } while( row != NULL);

  fclose(fp);

  return numOfRows;
}

1907 1908
static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
  TAOS_RES * res;
1909 1910
  TAOS_ROW row = NULL;
  int count = 0;
1911 1912

  res = taos_query(taos, "show databases;");
1913
  int32_t code = taos_errno(res);
1914

1915
  if (code != 0) {
1916
    errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
1917 1918 1919 1920 1921
    return -1;
  }

  TAOS_FIELD *fields = taos_fetch_fields(res);

1922
  while((row = taos_fetch_row(res)) != NULL) {
1923
    // sys database name : 'log'
1924 1925 1926 1927
    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log",
                fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) {
      continue;
    }
1928 1929 1930

    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
    if (dbInfos[count] == NULL) {
1931
      errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
1932 1933 1934
      return -1;
    }

1935 1936
    tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
            fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
1937
    formatTimestamp(dbInfos[count]->create_time,
1938 1939
            *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
            TSDB_TIME_PRECISION_MILLI);
1940
    dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
1941
    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
1942 1943
    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
1944
    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
1945

1946
    tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
1947
            fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
1948 1949 1950 1951 1952 1953 1954
    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
1955 1956
    dbInfos[count]->cachelast =
      (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));
1957

1958
    tstrncpy(dbInfos[count]->precision,
1959
            (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
1960
            fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
1961
    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
1962
    tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
1963
            fields[TSDB_SHOW_DB_STATUS_INDEX].bytes);
1964

1965 1966
    count++;
    if (count > MAX_DATABASE_COUNT) {
1967 1968
      errorPrint("%s() LN%d, The database count overflow than %d\n",
         __func__, __LINE__, MAX_DATABASE_COUNT);
1969 1970 1971 1972 1973 1974 1975
      break;
    }
  }

  return count;
}

1976 1977
static void printfDbInfoForQueryToFile(
        char* filename, SDbInfo* dbInfos, int index) {
1978

1979
  if (filename[0] == 0)
1980
      return;
1981 1982 1983

  FILE *fp = fopen(filename, "at");
  if (fp == NULL) {
1984
    errorPrint( "failed to open file: %s\n", filename);
1985
    return;
1986 1987 1988 1989 1990
  }

  fprintf(fp, "================ database[%d] ================\n", index);
  fprintf(fp, "name: %s\n", dbInfos->name);
  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
1991
  fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables);
1992
  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);
1993 1994
  fprintf(fp, "replica: %d\n", dbInfos->replica);
  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
1995 1996
  fprintf(fp, "days: %d\n", dbInfos->days);
  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);
1997 1998 1999 2000 2001 2002 2003
  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
  fprintf(fp, "comp: %d\n", dbInfos->comp);
2004 2005
  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);
  fprintf(fp, "precision: %s\n", dbInfos->precision);
2006
  fprintf(fp, "update: %d\n", dbInfos->update);
2007
  fprintf(fp, "status: %s\n", dbInfos->status);
2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021
  fprintf(fp, "\n");

  fclose(fp);
}

static void printfQuerySystemInfo(TAOS * taos) {
  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
  TAOS_RES* res;

  time_t t;
  struct tm* lt;
  time(&t);
  lt = localtime(&t);
2022 2023 2024 2025
  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
          lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
          lt->tm_sec);

2026 2027
  // show variables
  res = taos_query(taos, "show variables;");
2028
  //fetchResult(res, filename);
2029 2030 2031 2032 2033
  xDumpResultToFile(filename, res);

  // show dnodes
  res = taos_query(taos, "show dnodes;");
  xDumpResultToFile(filename, res);
2034
  //fetchResult(res, filename);
2035

2036 2037 2038 2039
  // show databases
  res = taos_query(taos, "show databases;");
  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
  if (dbInfos == NULL) {
2040
    errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
2041 2042 2043
    return;
  }
  int dbCount = getDbFromServer(taos, dbInfos);
2044 2045 2046 2047
  if (dbCount <= 0) {
      free(dbInfos);
      return;
  }
2048 2049

  for (int i = 0; i < dbCount; i++) {
2050
    // printf database info
2051
    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
2052

2053 2054 2055 2056
    // show db.vgroups
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);
2057

2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068
    // show db.stables
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);

    free(dbInfos[i]);
  }

  free(dbInfos);
}

2069
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
2070
        char* sqlstr, threadInfo *pThreadInfo)
2071
{
2072
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
2073 2074 2075

    char *url = "/rest/sql";

2076
    int bytes, sent, received, req_str_len, resp_len;
2077 2078 2079
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
2080

2081 2082 2083
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;

    request_buf = malloc(req_buf_len);
2084 2085 2086 2087
    if (NULL == request_buf) {
      errorPrint("%s", "ERROR, cannot allocate memory.\n");
      exit(EXIT_FAILURE);
    }
2088

2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};

    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
2100

2101 2102 2103 2104 2105 2106
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);

    char base64_buf[INPUT_BUF_LEN];
2107
#ifdef WINDOWS
2108
    WSADATA wsaData;
2109
    WSAStartup(MAKEWORD(2, 2), &wsaData);
2110 2111 2112 2113
    SOCKET sockfd;
#else
    int sockfd;
#endif
2114
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
2115
    if (sockfd < 0) {
2116
#ifdef WINDOWS
2117
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
2118
#endif
2119
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
2120
        free(request_buf);
2121
        ERROR_EXIT("ERROR opening socket");
2122
    }
2123

2124
    int retConn = connect(sockfd, (struct sockaddr *)pServAddr, sizeof(struct sockaddr));
2125 2126
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
2127
        free(request_buf);
2128
        ERROR_EXIT("ERROR connecting");
2129
    }
2130

2131 2132 2133
    memset(base64_buf, 0, INPUT_BUF_LEN);

    for (int n = 0, m = 0; n < userpass_buf_len;) {
2134
      uint32_t oct_a = n < userpass_buf_len ?
2135
        (unsigned char) userpass_buf[n++]:0;
2136
      uint32_t oct_b = n < userpass_buf_len ?
2137
        (unsigned char) userpass_buf[n++]:0;
2138
      uint32_t oct_c = n < userpass_buf_len ?
2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;

      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }

    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';

2151 2152
    debugPrint("%s() LN%d: auth string base64 encoded: %s\n",
            __func__, __LINE__, base64_buf);
2153 2154
    char *auth = base64_buf;

2155 2156 2157
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
2158 2159 2160 2161 2162
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
2163
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
2164

2165
    req_str_len = strlen(request_buf);
2166 2167
    sent = 0;
    do {
2168 2169 2170
#ifdef WINDOWS
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
#else
2171
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
2172
#endif
2173
        if (bytes < 0)
2174
            ERROR_EXIT("ERROR writing message to socket");
2175 2176 2177
        if (bytes == 0)
            break;
        sent+=bytes;
2178
    } while(sent < req_str_len);
2179

2180 2181
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
2182 2183
    received = 0;
    do {
2184 2185 2186
#ifdef WINDOWS
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
#else
2187
        bytes = read(sockfd, response_buf + received, resp_len - received);
2188
#endif
2189 2190
        if (bytes < 0) {
            free(request_buf);
2191
            ERROR_EXIT("ERROR reading response from socket");
2192
        }
2193 2194
        if (bytes == 0)
            break;
2195
        received += bytes;
2196
    } while(received < resp_len);
2197

2198 2199
    if (received == resp_len) {
        free(request_buf);
2200
        ERROR_EXIT("ERROR storing complete response from socket");
2201
    }
2202

2203
    response_buf[RESP_BUF_LEN - 1] = '\0';
2204 2205
    printf("Response:\n%s\n", response_buf);

2206 2207
    if (strlen(pThreadInfo->filePath) > 0) {
       appendResultBufToFile(response_buf, pThreadInfo);
2208 2209
    }

2210
    free(request_buf);
2211 2212 2213 2214
#ifdef WINDOWS
    closesocket(sockfd);
    WSACleanup();
#else
2215
    close(sockfd);
2216
#endif
2217 2218 2219 2220

    return 0;
}

2221
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
2222 2223
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
2224 2225
    errorPrint("%s() LN%d, calloc failed! size:%d\n",
        __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
2226 2227
    return NULL;
  }
2228

2229
  int    dataLen = 0;
2230 2231 2232
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);

2233 2234 2235
  return dataBuf;
}

2236
static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) {
2237 2238 2239 2240 2241 2242 2243 2244 2245
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
    printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
    return NULL;
  }

  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
  for (int i = 0; i < stbInfo->tagCount; i++) {
2246 2247
    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
2248
      if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
2249 2250
        printf("binary or nchar length overflow, max size:%u\n",
                (uint32_t)TSDB_MAX_BINARY_LEN);
2251 2252 2253
        tmfree(dataBuf);
        return NULL;
      }
2254

2255 2256
      int tagBufLen = stbInfo->tags[i].dataLen + 1;
      char* buf = (char*)calloc(tagBufLen, 1);
2257 2258 2259 2260 2261
      if (NULL == buf) {
        printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
        tmfree(dataBuf);
        return NULL;
      }
2262 2263 2264 2265 2266 2267 2268

      if (tableSeq % 2) {
        tstrncpy(buf, "beijing", tagBufLen);
      } else {
        tstrncpy(buf, "shanghai", tagBufLen);
      }
      //rand_string(buf, stbInfo->tags[i].dataLen);
2269 2270
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "\'%s\', ", buf);
2271
      tmfree(buf);
2272 2273 2274
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "int", strlen("int"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2275
              "%d, ", tableSeq);
2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bigint", strlen("bigint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "float", strlen("float"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_float());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "double", strlen("double"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_double());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "smallint", strlen("smallint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_smallint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "tinyint", strlen("tinyint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_tinyint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bool", strlen("bool"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_bool());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "timestamp", strlen("timestamp"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
2304 2305 2306 2307 2308 2309
    }  else {
      printf("No support data type: %s\n", stbInfo->tags[i].dataType);
      tmfree(dataBuf);
      return NULL;
    }
  }
2310

2311
  dataLen -= 2;
2312
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");
2313 2314 2315
  return dataBuf;
}

2316
static int calcRowLen(SSuperTable*  superTbls) {
2317 2318
  int colIndex;
  int  lenOfOneRow = 0;
2319

2320 2321
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2322

2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
2339
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2340
      lenOfOneRow += 42;
2341
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
2355

2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
2372
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2373 2374 2375 2376 2377 2378 2379 2380
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2381

2382 2383 2384 2385
  return 0;
}


2386 2387
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
2388
        int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) {
2389

2390
  char command[BUFFER_SIZE] = "\0";
2391 2392
  char limitBuf[100] = "\0";

2393
  TAOS_RES * res;
2394 2395 2396
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2397 2398

  if (offset >= 0) {
2399
    snprintf(limitBuf, 100, " limit %"PRId64" offset %"PRIu64"",
2400
            limit, offset);
2401 2402
  }

2403
  //get all child table name use cmd: select tbname from superTblName;
2404 2405
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s",
          dbName, sTblName, limitBuf);
2406

2407
  res = taos_query(taos, command);
2408 2409 2410 2411
  int32_t code = taos_errno(res);
  if (code != 0) {
    taos_free_result(res);
    taos_close(taos);
2412 2413
    errorPrint("%s() LN%d, failed to run command %s\n",
           __func__, __LINE__, command);
2414 2415 2416
    exit(-1);
  }

2417 2418
  int64_t childTblCount = (limit < 0)?10000:limit;
  int64_t count = 0;
2419 2420 2421 2422 2423 2424 2425 2426 2427 2428
  if (childTblName == NULL) {
    childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (NULL ==  childTblName) {
    taos_free_result(res);
        taos_close(taos);
        errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__);
        exit(-1);
    }
  }

2429
  char* pTblName = childTblName;
2430
  while((row = taos_fetch_row(res)) != NULL) {
2431
    int32_t* len = taos_fetch_lengths(res);
2432
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2433 2434 2435
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2436 2437
      char *tmp = realloc(childTblName,
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2438 2439 2440
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2441 2442
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2443 2444
      } else {
        // exit, if allocate more memory failed
2445 2446
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
               __func__, __LINE__, dbName, sTblName);
2447 2448 2449 2450 2451 2452
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2453
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2454
  }
2455

2456 2457 2458 2459 2460 2461 2462
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2463 2464
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
        char* sTblName, char** childTblNameOfSuperTbl,
2465
        int64_t* childTblCountOfSuperTbl) {
2466 2467 2468

    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
2469
            -1, 0);
2470 2471
}

2472 2473
static int getSuperTableFromServer(TAOS * taos, char* dbName,
        SSuperTable*  superTbls) {
2474

2475
  char command[BUFFER_SIZE] = "\0";
2476
  TAOS_RES * res;
2477 2478
  TAOS_ROW row = NULL;
  int count = 0;
2479

2480
  //get schema use cmd: describe superTblName;
2481
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2482
  res = taos_query(taos, command);
2483 2484 2485
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2486
    taos_free_result(res);
2487 2488 2489 2490 2491 2492
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
2493
  while((row = taos_fetch_row(res)) != NULL) {
2494 2495 2496
    if (0 == count) {
      count++;
      continue;
2497
    }
2498 2499

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510
      tstrncpy(superTbls->tags[tagIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->tags[tagIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2511
      tagIndex++;
2512
    } else {
2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523
      tstrncpy(superTbls->columns[columnIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->columns[columnIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2535
/*
2536
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
2537
    //get all child table name use cmd: select tbname from superTblName;
2538 2539 2540
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2541
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2542 2543
      return -1;
    }
2544
    getAllChildNameOfSuperTable(taos, dbName,
2545 2546 2547
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2548
  }
2549
  */
2550 2551 2552
  return 0;
}

2553 2554
static int createSuperTable(
        TAOS * taos, char* dbName,
2555
        SSuperTable*  superTbl) {
2556

2557
  char command[BUFFER_SIZE] = "\0";
2558

2559 2560 2561 2562 2563
  char cols[STRING_LEN] = "\0";
  int colIndex;
  int len = 0;

  int  lenOfOneRow = 0;
2564 2565 2566 2567 2568 2569 2570 2571 2572

  if (superTbl->columnCount == 0) {
    errorPrint("%s() LN%d, super table column count is %d\n",
            __func__, __LINE__, superTbl->columnCount);
    return -1;
  }

  for (colIndex = 0; colIndex < superTbl->columnCount; colIndex++) {
    char* dataType = superTbl->columns[colIndex].dataType;
2573

2574
    if (strcasecmp(dataType, "BINARY") == 0) {
2575 2576
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "BINARY",
2577 2578
          superTbl->columns[colIndex].dataLen);
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2579
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
2580 2581
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "NCHAR",
2582 2583
          superTbl->columns[colIndex].dataLen);
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
      lenOfOneRow += 22;
2602
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2603 2604
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
      lenOfOneRow += 42;
2605
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2606 2607 2608 2609
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
      lenOfOneRow += 21;
    } else {
      taos_close(taos);
2610 2611
      errorPrint("%s() LN%d, config error data type : %s\n",
         __func__, __LINE__, dataType);
2612 2613 2614 2615
      exit(-1);
    }
  }

2616 2617
  superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp
  //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbl[j].sTblName, g_Dbs.db[i].superTbl[j].columnCount, lenOfOneRow);
2618 2619

  // save for creating child table
2620 2621 2622 2623
  superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1);
  if (NULL == superTbl->colsOfCreateChildTable) {
    errorPrint("%s() LN%d, Failed when calloc, size:%d",
           __func__, __LINE__, len+1);
2624 2625 2626 2627
    taos_close(taos);
    exit(-1);
  }

2628
  snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
2629 2630
  verbosePrint("%s() LN%d: %s\n",
      __func__, __LINE__, superTbl->colsOfCreateChildTable);
2631

2632 2633 2634 2635 2636
  if (superTbl->tagCount == 0) {
    errorPrint("%s() LN%d, super table tag count is %d\n",
            __func__, __LINE__, superTbl->tagCount);
    return -1;
  }
2637

2638 2639 2640
  char tags[STRING_LEN] = "\0";
  int tagIndex;
  len = 0;
2641

2642 2643 2644 2645
  int lenOfTagOfOneRow = 0;
  len += snprintf(tags + len, STRING_LEN - len, "(");
  for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) {
    char* dataType = superTbl->tags[tagIndex].dataType;
2646

2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684
    if (strcasecmp(dataType, "BINARY") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
              "BINARY", superTbl->tags[tagIndex].dataLen);
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
              "NCHAR", superTbl->tags[tagIndex].dataLen);
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "INT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "BIGINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "SMALLINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "TINYINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "BOOL");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "FLOAT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "DOUBLE");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42;
    } else {
      taos_close(taos);
2685 2686
      errorPrint("%s() LN%d, config error tag type : %s\n",
         __func__, __LINE__, dataType);
2687
      exit(-1);
2688 2689
    }
  }
2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704

  len -= 2;
  len += snprintf(tags + len, STRING_LEN - len, ")");

  superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow;

  snprintf(command, BUFFER_SIZE,
          "create table if not exists %s.%s (ts timestamp%s) tags %s",
          dbName, superTbl->sTblName, cols, tags);
  if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
      errorPrint( "create supertable %s failed!\n\n",
              superTbl->sTblName);
      return -1;
  }
  debugPrint("create supertable %s success!\n\n", superTbl->sTblName);
2705 2706 2707
  return 0;
}

2708
static int createDatabasesAndStables() {
2709 2710 2711 2712
  TAOS * taos = NULL;
  int    ret = 0;
  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
  if (taos == NULL) {
2713
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2714
    return -1;
2715 2716 2717
  }
  char command[BUFFER_SIZE] = "\0";

2718
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2719 2720
    if (g_Dbs.db[i].drop) {
      sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
2721
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2722 2723 2724 2725
        taos_close(taos);
        return -1;
      }

2726
      int dataLen = 0;
2727
      dataLen += snprintf(command + dataLen,
2728
          BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2729

2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798
      if (g_Dbs.db[i].dbCfg.blocks > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
      }
      if (g_Dbs.db[i].dbCfg.cache > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
      }
      if (g_Dbs.db[i].dbCfg.days > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
      }
      if (g_Dbs.db[i].dbCfg.keep > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
      }
      if (g_Dbs.db[i].dbCfg.quorum > 1) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
      }
      if (g_Dbs.db[i].dbCfg.replica > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
      }
      if (g_Dbs.db[i].dbCfg.update > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
      }
      //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
      //  dataLen += snprintf(command + dataLen,
      //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
      //}
      if (g_Dbs.db[i].dbCfg.minRows > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
      }
      if (g_Dbs.db[i].dbCfg.maxRows > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
      }
      if (g_Dbs.db[i].dbCfg.comp > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
      }
      if (g_Dbs.db[i].dbCfg.walLevel > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
      }
      if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
      }
      if (g_Dbs.db[i].dbCfg.fsync > 0) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
                " fsync %d", g_Dbs.db[i].dbCfg.fsync);
      }
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                      "us", strlen("us")))) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
                " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
      }

      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
        taos_close(taos);
        errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
        return -1;
      }
      printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
2799 2800
    }

2801
    debugPrint("%s() LN%d supertbl count:%"PRIu64"\n",
2802
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
2803 2804 2805

    int validStbCount = 0;

2806
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
2807 2808 2809 2810 2811
      sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
              g_Dbs.db[i].superTbls[j].sTblName);
      ret = queryDbExec(taos, command, NO_INSERT_TYPE, true);

      if ((ret != 0) || (g_Dbs.db[i].drop)) {
2812
        ret = createSuperTable(taos, g_Dbs.db[i].dbName,
2813
                &g_Dbs.db[i].superTbls[j]);
2814

2815
        if (0 != ret) {
2816 2817
          errorPrint("create super table %d failed!\n\n", j);
          continue;
2818 2819 2820
        }
      }

2821
      ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
2822
                &g_Dbs.db[i].superTbls[j]);
2823 2824 2825
      if (0 != ret) {
        errorPrint("\nget super table %s.%s info failed!\n\n",
                g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName);
2826
        continue;
2827
      }
2828 2829

      validStbCount ++;
2830
    }
2831 2832

    g_Dbs.db[i].superTblCount = validStbCount;
2833 2834 2835 2836 2837 2838
  }

  taos_close(taos);
  return 0;
}

2839 2840
static void* createTable(void *sarg)
{
2841 2842
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
2843 2844 2845

  int64_t  lastPrintTime = taosGetTimestampMs();

2846
  int buff_len;
2847
  buff_len = BUFFER_SIZE / 8;
2848

2849
  char *buffer = calloc(buff_len, 1);
2850
  if (buffer == NULL) {
2851
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
2852 2853
    exit(-1);
  }
2854 2855 2856

  int len = 0;
  int batchNum = 0;
2857

2858
  verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n",
2859
          __func__, __LINE__,
2860
          pThreadInfo->start_table_from, pThreadInfo->end_table_to);
2861

2862
  for (uint64_t i = pThreadInfo->start_table_from;
2863
          i <= pThreadInfo->end_table_to; i++) {
2864
    if (0 == g_Dbs.use_metric) {
2865
      snprintf(buffer, buff_len,
2866
              "create table if not exists %s.%s%"PRIu64" %s;",
2867
              pThreadInfo->db_name,
2868
              g_args.tb_prefix, i,
2869
              pThreadInfo->cols);
2870
    } else {
2871 2872 2873
      if (superTblInfo == NULL) {
        errorPrint("%s() LN%d, use metric, but super table info is NULL\n",
                  __func__, __LINE__);
2874
        free(buffer);
2875 2876 2877 2878 2879 2880 2881 2882 2883 2884
        exit(-1);
      } else {
        if (0 == len) {
          batchNum = 0;
          memset(buffer, 0, buff_len);
          len += snprintf(buffer + len,
                  buff_len - len, "create table ");
        }
        char* tagsValBuf = NULL;
        if (0 == superTblInfo->tagSource) {
2885
          tagsValBuf = generateTagVaulesForStb(superTblInfo, i);
2886 2887 2888 2889 2890 2891 2892 2893 2894 2895
        } else {
          tagsValBuf = getTagValueFromTagSample(
                  superTblInfo,
                  i % superTblInfo->tagSampleCount);
        }
        if (NULL == tagsValBuf) {
          free(buffer);
          return NULL;
        }
        len += snprintf(buffer + len,
2896
                buff_len - len,
2897
                "if not exists %s.%s%"PRIu64" using %s.%s tags %s ",
2898 2899
                pThreadInfo->db_name, superTblInfo->childTblPrefix,
                i, pThreadInfo->db_name,
2900 2901 2902 2903
                superTblInfo->sTblName, tagsValBuf);
        free(tagsValBuf);
        batchNum++;
        if ((batchNum < superTblInfo->batchCreateTableNum)
2904
                && ((buff_len - len)
2905 2906 2907
                    >= (superTblInfo->lenOfTagOfOneRow + 256))) {
          continue;
        }
2908 2909 2910 2911
      }
    }

    len = 0;
2912
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)){
2913
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2914
      free(buffer);
2915 2916 2917 2918 2919
      return NULL;
    }

    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
2920
      printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n",
2921
              pThreadInfo->threadID, pThreadInfo->start_table_from, i);
2922 2923 2924
      lastPrintTime = currentPrintTime;
    }
  }
2925

2926
  if (0 != len) {
2927
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)) {
2928
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2929
    }
2930
  }
2931

2932 2933 2934 2935
  free(buffer);
  return NULL;
}

2936
static int startMultiThreadCreateChildTable(
2937
        char* cols, int threads, uint64_t startFrom, int64_t ntables,
2938
        char* db_name, SSuperTable* superTblInfo) {
2939

2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  threadInfo *infos = malloc(threads * sizeof(threadInfo));

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

2952
  int64_t a = ntables / threads;
2953 2954 2955 2956 2957
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

2958
  int64_t b = 0;
2959
  b = ntables % threads;
2960

2961
  for (int64_t i = 0; i < threads; i++) {
2962 2963 2964 2965
    threadInfo *pThreadInfo = infos + i;
    pThreadInfo->threadID = i;
    tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE);
    pThreadInfo->superTblInfo = superTblInfo;
2966
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
2967
    pThreadInfo->taos = taos_connect(
2968 2969 2970 2971 2972
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
2973
    if (pThreadInfo->taos == NULL) {
2974 2975
      errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n",
         __func__, __LINE__, taos_errstr(NULL));
2976
      free(pids);
2977
      free(infos);
2978 2979
      return -1;
    }
2980

2981 2982 2983 2984 2985 2986 2987 2988
    pThreadInfo->start_table_from = startFrom;
    pThreadInfo->ntables = i<b?a+1:a;
    pThreadInfo->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
    startFrom = pThreadInfo->end_table_to + 1;
    pThreadInfo->use_metric = true;
    pThreadInfo->cols = cols;
    pThreadInfo->minDelay = UINT64_MAX;
    pthread_create(pids + i, NULL, createTable, pThreadInfo);
2989
  }
2990

2991 2992 2993 2994 2995
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
2996 2997
    threadInfo *pThreadInfo = infos + i;
    taos_close(pThreadInfo->taos);
2998 2999 3000
  }

  free(pids);
3001
  free(infos);
3002 3003

  return 0;
3004 3005
}

3006 3007 3008 3009
static void createChildTables() {
    char tblColsBuf[MAX_SQL_SIZE];
    int len;

3010
  for (int i = 0; i < g_Dbs.dbCount; i++) {
3011 3012 3013 3014
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
          // with super table
        for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
3015
          if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
3016 3017
            continue;
          }
3018

3019 3020
          verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
                  g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
3021
          uint64_t startFrom = 0;
3022 3023
          g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;

3024
          verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n",
3025 3026 3027 3028 3029 3030 3031 3032
                  __func__, __LINE__, g_totalChildTables, startFrom);
          startMultiThreadCreateChildTable(
                g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
                g_Dbs.threadCountByCreateTbl,
                startFrom,
                g_Dbs.db[i].superTbls[j].childTblCount,
                g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
        }
3033 3034
      }
    } else {
3035 3036
      // normal table
      len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
3037
      for (int j = 0; j < g_args.num_of_CPR; j++) {
3038 3039 3040 3041
          if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
                  || (strncasecmp(g_args.datatype[j],
                      "NCHAR", strlen("NCHAR")) == 0)) {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
3042
                      ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary);
3043 3044 3045 3046 3047 3048
          } else {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                      ", COL%d %s", j, g_args.datatype[j]);
          }
          len = strlen(tblColsBuf);
      }
3049

3050
      snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");
3051

3052
      verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n",
3053 3054 3055 3056 3057 3058 3059 3060 3061
              __func__, __LINE__,
              g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
      startMultiThreadCreateChildTable(
            tblColsBuf,
            g_Dbs.threadCountByCreateTbl,
            0,
            g_args.num_of_tables,
            g_Dbs.db[i].dbName,
            NULL);
3062
    }
3063 3064 3065 3066 3067 3068
  }
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3069
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
3070 3071 3072
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
3073

3074 3075
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
3076 3077
    printf("Failed to open tags file: %s, reason:%s\n",
            superTblInfo->tagsFile, strerror(errno));
3078 3079 3080 3081 3082 3083 3084
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
3085

3086 3087 3088 3089 3090 3091 3092 3093 3094
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

3095
  while((readLen = tgetline(&line, &n, fp)) != -1) {
3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
3108 3109
      char *tmp = realloc(tagDataBuf,
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
3110 3111 3112
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
3113 3114
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

int readSampleFromJsonFileToMem(SSuperTable  * superTblInfo) {
  // TODO
  return 0;
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3142 3143
static int readSampleFromCsvFileToMem(
        SSuperTable* superTblInfo) {
3144 3145 3146 3147
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
3148

3149 3150
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
3151
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
3152 3153 3154
              superTblInfo->sampleFile, strerror(errno));
      return -1;
  }
3155

3156
  assert(superTblInfo->sampleDataBuf);
3157 3158
  memset(superTblInfo->sampleDataBuf, 0,
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
3159
  while(1) {
3160
    readLen = tgetline(&line, &n, fp);
3161 3162
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
3163
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
3164
                superTblInfo->sampleFile, strerror(errno));
3165
        fclose(fp);
3166 3167 3168 3169
        return -1;
      }
      continue;
    }
3170

3171 3172 3173 3174 3175 3176 3177 3178 3179
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
3180
      printf("sample row len[%d] overflow define schema len[%"PRIu64"], so discard this row\n",
3181
              (int32_t)readLen, superTblInfo->lenOfOneRow);
3182 3183 3184
      continue;
    }

3185 3186
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
          line, readLen);
3187 3188 3189 3190 3191 3192 3193
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

3194
  fclose(fp);
3195 3196 3197 3198
  tmfree(line);
  return 0;
}

3199 3200
static bool getColumnAndTagTypeFromInsertJsonFile(
        cJSON* stbInfo, SSuperTable* superTbls) {
3201
  bool  ret = false;
3202

3203
  // columns
3204 3205
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
3206
    printf("ERROR: failed to read json, columns not found\n");
3207 3208 3209 3210 3211 3212
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
3213

3214
  int columnSize = cJSON_GetArraySize(columns);
3215
  if ((columnSize + 1/* ts */) > MAX_COLUMN_COUNT) {
3216 3217
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
            __func__, __LINE__, MAX_COLUMN_COUNT);
3218 3219 3220 3221 3222 3223
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
3224

3225
  //superTbls->columnCount = columnSize;
3226 3227 3228 3229 3230 3231 3232
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
3233
      count = countObj->valueint;
3234
    } else if (countObj && countObj->type != cJSON_Number) {
3235 3236
      errorPrint("%s() LN%d, failed to read json, column count not found\n",
          __func__, __LINE__);
3237 3238 3239 3240 3241
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3242
    // column info
3243 3244
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
3245 3246 3247 3248
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d: failed to read json, column type not found\n",
          __func__, __LINE__);
3249 3250
      goto PARSE_OVER;
    }
3251 3252
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3253

3254 3255
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3256
      columnCase.dataLen = dataLen->valueint;
3257
    } else if (dataLen && dataLen->type != cJSON_Number) {
3258 3259
      debugPrint("%s() LN%d: failed to read json, column len not found\n",
          __func__, __LINE__);
3260 3261 3262 3263
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
3264

3265
    for (int n = 0; n < count; ++n) {
3266 3267 3268
      tstrncpy(superTbls->columns[index].dataType,
              columnCase.dataType, MAX_TB_NAME_SIZE);
      superTbls->columns[index].dataLen = columnCase.dataLen;
3269 3270
      index++;
    }
3271
  }
3272

3273 3274
  if ((index + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n",
3275 3276 3277 3278
            __func__, __LINE__, MAX_COLUMN_COUNT);
    goto PARSE_OVER;
  }

3279
  superTbls->columnCount = index;
3280

3281 3282
  count = 1;
  index = 0;
3283
  // tags
3284 3285
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
3286 3287
    errorPrint("%s() LN%d, failed to read json, tags not found\n",
        __func__, __LINE__);
3288 3289
    goto PARSE_OVER;
  }
3290

3291 3292
  int tagSize = cJSON_GetArraySize(tags);
  if (tagSize > MAX_TAG_COUNT) {
3293 3294
    errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n",
        __func__, __LINE__, MAX_TAG_COUNT);
3295 3296
    goto PARSE_OVER;
  }
3297 3298

  //superTbls->tagCount = tagSize;
3299 3300 3301
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
3302

3303 3304 3305
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
3306
      count = countObj->valueint;
3307
    } else if (countObj && countObj->type != cJSON_Number) {
3308
      printf("ERROR: failed to read json, column count not found\n");
3309 3310 3311 3312 3313
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3314
    // column info
3315 3316
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
3317 3318 3319 3320
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d, failed to read json, tag type not found\n",
          __func__, __LINE__);
3321 3322
      goto PARSE_OVER;
    }
3323
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3324

3325 3326
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3327
      columnCase.dataLen = dataLen->valueint;
3328
    } else if (dataLen && dataLen->type != cJSON_Number) {
3329 3330
      errorPrint("%s() LN%d, failed to read json, column len not found\n",
          __func__, __LINE__);
3331 3332 3333
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
3334 3335
    }

3336
    for (int n = 0; n < count; ++n) {
3337 3338
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType,
          MAX_TB_NAME_SIZE);
3339
      superTbls->tags[index].dataLen = columnCase.dataLen;
3340 3341
      index++;
    }
3342
  }
3343 3344

  if (index > MAX_TAG_COUNT) {
3345
    errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n",
3346 3347 3348 3349
        __func__, __LINE__, MAX_TAG_COUNT);
    goto PARSE_OVER;
  }

3350 3351
  superTbls->tagCount = index;

3352 3353 3354
  if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n",
        __func__, __LINE__, MAX_COLUMN_COUNT);
3355 3356
    goto PARSE_OVER;
  }
3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367
  ret = true;

PARSE_OVER:
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3368
    tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3369 3370 3371 3372
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3373
    tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE);
3374
  } else if (!host) {
3375
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3376
  } else {
3377
    printf("ERROR: failed to read json, host not found\n");
3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_Dbs.port = port->valueint;
  } else if (!port) {
    g_Dbs.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3390
    tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE);
3391
  } else if (!user) {
3392
    tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE);
3393 3394 3395 3396
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3397
    tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE);
3398
  } else if (!password) {
3399
    tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE);
3400 3401 3402 3403
  }

  cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
  if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
3404
    tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
3405
  } else if (!resultfile) {
3406
    tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
3407 3408 3409 3410 3411 3412 3413 3414
  }

  cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
  if (threads && threads->type == cJSON_Number) {
    g_Dbs.threadCount = threads->valueint;
  } else if (!threads) {
    g_Dbs.threadCount = 1;
  } else {
3415
    printf("ERROR: failed to read json, threads not found\n");
3416
    goto PARSE_OVER;
3417 3418
  }

3419 3420 3421 3422
  cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
  if (threads2 && threads2->type == cJSON_Number) {
    g_Dbs.threadCountByCreateTbl = threads2->valueint;
  } else if (!threads2) {
3423
    g_Dbs.threadCountByCreateTbl = 1;
3424
  } else {
3425 3426
    errorPrint("%s() LN%d, failed to read json, threads2 not found\n",
            __func__, __LINE__);
3427
    goto PARSE_OVER;
3428
  }
3429

3430 3431
  cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
  if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
3432 3433 3434 3435 3436
    if (gInsertInterval->valueint <0) {
      errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n",
            __func__, __LINE__);
      goto PARSE_OVER;
    }
3437 3438 3439 3440
    g_args.insert_interval = gInsertInterval->valueint;
  } else if (!gInsertInterval) {
    g_args.insert_interval = 0;
  } else {
3441 3442
    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
        __func__, __LINE__);
3443 3444
    goto PARSE_OVER;
  }
3445

3446 3447
  cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows");
  if (interlaceRows && interlaceRows->type == cJSON_Number) {
3448 3449 3450 3451 3452 3453
    if (interlaceRows->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;

    }
3454 3455
    g_args.interlace_rows = interlaceRows->valueint;
  } else if (!interlaceRows) {
3456
    g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3457
  } else {
3458 3459
    errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
        __func__, __LINE__);
3460
    goto PARSE_OVER;
3461
  }
3462

3463 3464
  cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
  if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
3465 3466 3467 3468 3469
    if (maxSqlLen->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
    }
3470 3471
    g_args.max_sql_len = maxSqlLen->valueint;
  } else if (!maxSqlLen) {
3472
    g_args.max_sql_len = (1024*1024);
3473
  } else {
3474 3475
    errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
        __func__, __LINE__);
3476 3477 3478 3479 3480
    goto PARSE_OVER;
  }

  cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
  if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
3481
    if (numRecPerReq->valueint <= 0) {
3482 3483 3484
      errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
3485
    } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) {
3486 3487 3488 3489 3490
      printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n",
              numRecPerReq->valueint, MAX_RECORDS_PER_REQ);
      printf("        number of records per request value will be set to %d\n\n",
              MAX_RECORDS_PER_REQ);
      prompt();
3491
      numRecPerReq->valueint = MAX_RECORDS_PER_REQ;
3492
    }
3493 3494
    g_args.num_of_RPR = numRecPerReq->valueint;
  } else if (!numRecPerReq) {
3495
    g_args.num_of_RPR = MAX_RECORDS_PER_REQ;
3496
  } else {
3497 3498
    errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n",
        __func__, __LINE__);
3499 3500 3501
    goto PARSE_OVER;
  }

3502
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3503
  if (answerPrompt
3504 3505
          && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3506 3507 3508 3509 3510 3511 3512 3513
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
    } else {
      g_args.answer_yes = false;
    }
  } else if (!answerPrompt) {
3514
    g_args.answer_yes = true;   // default is no, mean answer_yes.
3515
  } else {
3516
    errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n");
3517
    goto PARSE_OVER;
3518
  }
3519

3520 3521 3522 3523 3524 3525 3526 3527 3528 3529
  // rows per table need be less than insert batch
  if (g_args.interlace_rows > g_args.num_of_RPR) {
      printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
              g_args.interlace_rows, g_args.num_of_RPR);
      printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
              g_args.num_of_RPR);
      prompt();
      g_args.interlace_rows = g_args.num_of_RPR;
  }

3530 3531
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (!dbs || dbs->type != cJSON_Array) {
3532
    printf("ERROR: failed to read json, databases not found\n");
3533 3534 3535 3536 3537
    goto PARSE_OVER;
  }

  int dbSize = cJSON_GetArraySize(dbs);
  if (dbSize > MAX_DB_COUNT) {
3538
    errorPrint(
3539 3540
            "ERROR: failed to read json, databases size overflow, max database is %d\n",
            MAX_DB_COUNT);
3541 3542 3543 3544 3545 3546 3547 3548
    goto PARSE_OVER;
  }

  g_Dbs.dbCount = dbSize;
  for (int i = 0; i < dbSize; ++i) {
    cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
    if (dbinfos == NULL) continue;

3549
    // dbinfo
3550 3551
    cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
    if (!dbinfo || dbinfo->type != cJSON_Object) {
3552
      printf("ERROR: failed to read json, dbinfo not found\n");
3553 3554
      goto PARSE_OVER;
    }
3555

3556 3557
    cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
    if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
3558
      printf("ERROR: failed to read json, db name not found\n");
3559 3560
      goto PARSE_OVER;
    }
3561
    tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE);
3562 3563 3564

    cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
    if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
3565 3566
      if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) {
        g_Dbs.db[i].drop = true;
3567
      } else {
3568
        g_Dbs.db[i].drop = false;
3569
      }
3570
    } else if (!drop) {
3571
      g_Dbs.db[i].drop = g_args.drop_database;
3572
    } else {
3573 3574
      errorPrint("%s() LN%d, failed to read json, drop input mistake\n",
              __func__, __LINE__);
3575 3576 3577 3578
      goto PARSE_OVER;
    }

    cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
3579 3580 3581 3582
    if (precision && precision->type == cJSON_String
            && precision->valuestring != NULL) {
      tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
              MAX_DB_NAME_SIZE);
3583
    } else if (!precision) {
3584
      //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
3585 3586
      memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE);
    } else {
3587
      printf("ERROR: failed to read json, precision not found\n");
3588 3589 3590 3591 3592 3593 3594 3595 3596
      goto PARSE_OVER;
    }

    cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
    if (update && update->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.update = update->valueint;
    } else if (!update) {
      g_Dbs.db[i].dbCfg.update = -1;
    } else {
3597
      printf("ERROR: failed to read json, update not found\n");
3598 3599 3600 3601 3602 3603 3604 3605 3606
      goto PARSE_OVER;
    }

    cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
    if (replica && replica->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.replica = replica->valueint;
    } else if (!replica) {
      g_Dbs.db[i].dbCfg.replica = -1;
    } else {
3607
      printf("ERROR: failed to read json, replica not found\n");
3608 3609 3610 3611 3612 3613 3614 3615 3616
      goto PARSE_OVER;
    }

    cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
    if (keep && keep->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.keep = keep->valueint;
    } else if (!keep) {
      g_Dbs.db[i].dbCfg.keep = -1;
    } else {
3617
     printf("ERROR: failed to read json, keep not found\n");
3618 3619
     goto PARSE_OVER;
    }
3620

3621 3622 3623 3624 3625 3626
    cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
    if (days && days->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.days = days->valueint;
    } else if (!days) {
      g_Dbs.db[i].dbCfg.days = -1;
    } else {
3627
     printf("ERROR: failed to read json, days not found\n");
3628 3629
     goto PARSE_OVER;
    }
3630

3631 3632 3633 3634 3635 3636
    cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
    if (cache && cache->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cache = cache->valueint;
    } else if (!cache) {
      g_Dbs.db[i].dbCfg.cache = -1;
    } else {
3637
     printf("ERROR: failed to read json, cache not found\n");
3638 3639
     goto PARSE_OVER;
    }
3640

3641 3642 3643 3644 3645 3646
    cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
    if (blocks && blocks->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
    } else if (!blocks) {
      g_Dbs.db[i].dbCfg.blocks = -1;
    } else {
3647
     printf("ERROR: failed to read json, block not found\n");
3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664
     goto PARSE_OVER;
    }

    //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
    //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
    //} else if (!maxtablesPerVnode) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
    //} else {
    // printf("failed to read json, maxtablesPerVnode not found");
    // goto PARSE_OVER;
    //}

    cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
    if (minRows && minRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
    } else if (!minRows) {
3665
      g_Dbs.db[i].dbCfg.minRows = 0;    // 0 means default
3666
    } else {
3667
     printf("ERROR: failed to read json, minRows not found\n");
3668 3669 3670 3671 3672 3673 3674
     goto PARSE_OVER;
    }

    cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
    if (maxRows && maxRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
    } else if (!maxRows) {
3675
      g_Dbs.db[i].dbCfg.maxRows = 0;    // 0 means default
3676
    } else {
3677
     printf("ERROR: failed to read json, maxRows not found\n");
3678 3679 3680 3681 3682 3683 3684 3685 3686
     goto PARSE_OVER;
    }

    cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
    if (comp && comp->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.comp = comp->valueint;
    } else if (!comp) {
      g_Dbs.db[i].dbCfg.comp = -1;
    } else {
3687
     printf("ERROR: failed to read json, comp not found\n");
3688 3689 3690 3691 3692 3693 3694 3695 3696
     goto PARSE_OVER;
    }

    cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
    if (walLevel && walLevel->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
    } else if (!walLevel) {
      g_Dbs.db[i].dbCfg.walLevel = -1;
    } else {
3697
     printf("ERROR: failed to read json, walLevel not found\n");
3698 3699 3700
     goto PARSE_OVER;
    }

3701 3702 3703 3704 3705 3706
    cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
    if (cacheLast && cacheLast->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
    } else if (!cacheLast) {
      g_Dbs.db[i].dbCfg.cacheLast = -1;
    } else {
3707
     printf("ERROR: failed to read json, cacheLast not found\n");
3708 3709 3710
     goto PARSE_OVER;
    }

3711 3712 3713 3714
    cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
    if (quorum && quorum->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
    } else if (!quorum) {
3715
      g_Dbs.db[i].dbCfg.quorum = 1;
3716
    } else {
3717
     printf("failed to read json, quorum input mistake");
3718 3719 3720 3721 3722 3723 3724 3725 3726
     goto PARSE_OVER;
    }

    cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
    if (fsync && fsync->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
    } else if (!fsync) {
      g_Dbs.db[i].dbCfg.fsync = -1;
    } else {
3727 3728 3729
      errorPrint("%s() LN%d, failed to read json, fsync input mistake\n",
              __func__, __LINE__);
      goto PARSE_OVER;
3730
    }
3731

3732
    // super_talbes
3733 3734
    cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
    if (!stables || stables->type != cJSON_Array) {
3735 3736
      errorPrint("%s() LN%d, failed to read json, super_tables not found\n",
              __func__, __LINE__);
3737
      goto PARSE_OVER;
3738 3739
    }

3740 3741
    int stbSize = cJSON_GetArraySize(stables);
    if (stbSize > MAX_SUPER_TABLE_COUNT) {
3742
      errorPrint(
3743 3744
              "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n",
              __func__, __LINE__, MAX_SUPER_TABLE_COUNT);
3745 3746 3747 3748 3749 3750 3751
      goto PARSE_OVER;
    }

    g_Dbs.db[i].superTblCount = stbSize;
    for (int j = 0; j < stbSize; ++j) {
      cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
      if (stbInfo == NULL) continue;
3752 3753

      // dbinfo
3754
      cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
3755 3756
      if (!stbName || stbName->type != cJSON_String
              || stbName->valuestring == NULL) {
3757 3758
        errorPrint("%s() LN%d, failed to read json, stb name not found\n",
                __func__, __LINE__);
3759 3760
        goto PARSE_OVER;
      }
3761 3762
      tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring,
              MAX_TB_NAME_SIZE);
3763

3764 3765
      cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
      if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
3766
        printf("ERROR: failed to read json, childtable_prefix not found\n");
3767 3768
        goto PARSE_OVER;
      }
3769 3770
      tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring,
              MAX_DB_NAME_SIZE);
3771

3772
      cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table");
3773 3774 3775
      if (autoCreateTbl
              && autoCreateTbl->type == cJSON_String
              && autoCreateTbl->valuestring != NULL) {
3776 3777
        if ((0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3))
            && (TBL_ALREADY_EXISTS != g_Dbs.db[i].superTbls[j].childTblExists)) {
3778 3779 3780 3781 3782 3783 3784 3785 3786
          g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
        } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        } else {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        }
      } else if (!autoCreateTbl) {
        g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      } else {
3787
        printf("ERROR: failed to read json, auto_create_table not found\n");
3788 3789
        goto PARSE_OVER;
      }
3790

3791 3792 3793 3794
      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
      } else if (!batchCreateTbl) {
3795
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
3796
      } else {
3797
        printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
3798
        goto PARSE_OVER;
3799
      }
3800 3801

      cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
3802 3803 3804
      if (childTblExists
              && childTblExists->type == cJSON_String
              && childTblExists->valuestring != NULL) {
3805 3806
        if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3))
            && (g_Dbs.db[i].drop == false)) {
3807
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
3808 3809
        } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2)
              || (g_Dbs.db[i].drop == true))) {
3810 3811 3812 3813 3814 3815 3816
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        } else {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        }
      } else if (!childTblExists) {
        g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
      } else {
3817 3818
        errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n",
                __func__, __LINE__);
3819 3820
        goto PARSE_OVER;
      }
3821

3822 3823 3824 3825
      if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      }

3826 3827
      cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
      if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
3828
        errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n",
3829
                __func__, __LINE__);
3830 3831 3832 3833 3834
        goto PARSE_OVER;
      }
      g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

      cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
3835 3836 3837 3838
      if (dataSource && dataSource->type == cJSON_String
              && dataSource->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
                dataSource->valuestring, MAX_DB_NAME_SIZE);
3839
      } else if (!dataSource) {
3840
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE);
3841
      } else {
3842 3843
        errorPrint("%s() LN%d, failed to read json, data_source not found\n",
            __func__, __LINE__);
3844 3845 3846
        goto PARSE_OVER;
      }

3847
      cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest
3848 3849 3850 3851
      if (insertMode && insertMode->type == cJSON_String
              && insertMode->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode,
                insertMode->valuestring, MAX_DB_NAME_SIZE);
3852
      } else if (!insertMode) {
3853
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE);
3854
      } else {
3855
        printf("ERROR: failed to read json, insert_mode not found\n");
3856 3857 3858
        goto PARSE_OVER;
      }

3859
      cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
3860 3861
      if ((childTbl_limit) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3862 3863 3864 3865 3866 3867
        if (childTbl_limit->type != cJSON_Number) {
            printf("ERROR: failed to read json, childtable_limit\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
      } else {
3868
        g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid.
3869 3870 3871
      }

      cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
3872 3873
      if ((childTbl_offset) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3874 3875
        if ((childTbl_offset->type != cJSON_Number)
                || (0 > childTbl_offset->valueint)) {
3876 3877 3878 3879 3880 3881 3882 3883
            printf("ERROR: failed to read json, childtable_offset\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblOffset = 0;
      }

3884 3885
      cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
      if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
3886 3887
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                ts->valuestring, MAX_DB_NAME_SIZE);
3888
      } else if (!ts) {
3889 3890
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                "now", MAX_DB_NAME_SIZE);
3891
      } else {
3892
        printf("ERROR: failed to read json, start_timestamp not found\n");
3893 3894
        goto PARSE_OVER;
      }
3895

3896 3897 3898 3899
      cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
      if (timestampStep && timestampStep->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
      } else if (!timestampStep) {
3900
        g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
3901
      } else {
3902
        printf("ERROR: failed to read json, timestamp_step not found\n");
3903 3904 3905 3906
        goto PARSE_OVER;
      }

      cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
3907 3908
      if (sampleFormat && sampleFormat->type
              == cJSON_String && sampleFormat->valuestring != NULL) {
3909 3910
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
                sampleFormat->valuestring, MAX_DB_NAME_SIZE);
3911
      } else if (!sampleFormat) {
3912
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE);
3913
      } else {
3914
        printf("ERROR: failed to read json, sample_format not found\n");
3915
        goto PARSE_OVER;
3916
      }
3917

3918
      cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
3919 3920
      if (sampleFile && sampleFile->type == cJSON_String
          && sampleFile->valuestring != NULL) {
3921 3922
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
                sampleFile->valuestring, MAX_FILE_NAME_LEN);
3923 3924 3925
      } else if (!sampleFile) {
        memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
      } else {
3926
        printf("ERROR: failed to read json, sample_file not found\n");
3927
        goto PARSE_OVER;
3928
      }
3929

3930
      cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
3931 3932
      if ((tagsFile && tagsFile->type == cJSON_String)
              && (tagsFile->valuestring != NULL)) {
3933 3934
        tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
                tagsFile->valuestring, MAX_FILE_NAME_LEN);
3935 3936 3937 3938 3939 3940 3941 3942 3943
        if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
          g_Dbs.db[i].superTbls[j].tagSource = 0;
        } else {
          g_Dbs.db[i].superTbls[j].tagSource = 1;
        }
      } else if (!tagsFile) {
        memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
        g_Dbs.db[i].superTbls[j].tagSource = 0;
      } else {
3944
        printf("ERROR: failed to read json, tags_file not found\n");
3945 3946
        goto PARSE_OVER;
      }
3947

3948 3949 3950 3951 3952
      cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
      if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
        int32_t len = maxSqlLen->valueint;
        if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
          len = TSDB_MAX_ALLOWED_SQL_LEN;
3953 3954
        } else if (len < 5) {
          len = 5;
3955
        }
3956 3957
        g_Dbs.db[i].superTbls[j].maxSqlLen = len;
      } else if (!maxSqlLen) {
3958
        g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len;
3959
      } else {
3960 3961
        errorPrint("%s() LN%d, failed to read json, maxSqlLen input mistake\n",
            __func__, __LINE__);
3962
        goto PARSE_OVER;
3963
      }
3964
/*
3965 3966 3967 3968 3969
      cJSON *multiThreadWriteOneTbl =
          cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
      if (multiThreadWriteOneTbl
              && multiThreadWriteOneTbl->type == cJSON_String
              && multiThreadWriteOneTbl->valuestring != NULL) {
3970 3971 3972 3973
        if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
        } else {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
3974
        }
3975 3976 3977
      } else if (!multiThreadWriteOneTbl) {
        g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
      } else {
3978
        printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
3979 3980
        goto PARSE_OVER;
      }
3981
*/
3982 3983
      cJSON* interlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows");
      if (interlaceRows && interlaceRows->type == cJSON_Number) {
3984 3985 3986 3987 3988
        if (interlaceRows->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n",
            __func__, __LINE__);
          goto PARSE_OVER;
        }
3989
        g_Dbs.db[i].superTbls[j].interlaceRows = interlaceRows->valueint;
3990 3991
        // rows per table need be less than insert batch
        if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) {
3992
          printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
3993 3994 3995
                  i, j,
                  g_Dbs.db[i].superTbls[j].interlaceRows,
                  g_args.num_of_RPR);
3996
          printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
3997
                  g_args.num_of_RPR);
3998
          prompt();
3999 4000
          g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR;
        }
4001 4002
      } else if (!interlaceRows) {
        g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
4003
      } else {
4004 4005 4006
        errorPrint(
                "%s() LN%d, failed to read json, interlace rows input mistake\n",
                __func__, __LINE__);
4007
        goto PARSE_OVER;
4008
      }
4009 4010 4011

      cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
      if (disorderRatio && disorderRatio->type == cJSON_Number) {
4012 4013 4014 4015 4016 4017
        if (disorderRatio->valueint > 50)
          disorderRatio->valueint = 50;

        if (disorderRatio->valueint < 0)
          disorderRatio->valueint = 0;

4018 4019 4020 4021
        g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
      } else if (!disorderRatio) {
        g_Dbs.db[i].superTbls[j].disorderRatio = 0;
      } else {
4022
        printf("ERROR: failed to read json, disorderRatio not found\n");
4023
        goto PARSE_OVER;
4024
      }
4025 4026 4027 4028 4029 4030 4031

      cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
      if (disorderRange && disorderRange->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
      } else if (!disorderRange) {
        g_Dbs.db[i].superTbls[j].disorderRange = 1000;
      } else {
4032
        printf("ERROR: failed to read json, disorderRange not found\n");
4033 4034
        goto PARSE_OVER;
      }
4035

4036 4037
      cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
      if (insertRows && insertRows->type == cJSON_Number) {
4038 4039 4040 4041 4042
        if (insertRows->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4043 4044 4045 4046
        g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
      } else if (!insertRows) {
        g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
      } else {
4047 4048
        errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
4049 4050 4051 4052 4053 4054
        goto PARSE_OVER;
      }

      cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
      if (insertInterval && insertInterval->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
4055 4056 4057 4058 4059
        if (insertInterval->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4060
      } else if (!insertInterval) {
4061
        verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n",
4062 4063 4064
                __func__, __LINE__, g_args.insert_interval);
        g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
      } else {
4065 4066
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
4067 4068 4069
        goto PARSE_OVER;
      }

4070 4071
      int retVal = getColumnAndTagTypeFromInsertJsonFile(
              stbInfo, &g_Dbs.db[i].superTbls[j]);
4072 4073
      if (false == retVal) {
        goto PARSE_OVER;
4074 4075
      }
    }
4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088
  }

  ret = true;

PARSE_OVER:
  return ret;
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
4089
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
4090 4091 4092 4093
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
4094
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE);
4095
  } else if (!host) {
4096
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
4097
  } else {
4098
    printf("ERROR: failed to read json, host not found\n");
4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
4111
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE);
4112
  } else if (!user) {
4113
    tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ;
4114 4115 4116 4117
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
4118
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE);
4119
  } else if (!password) {
4120
    tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);;
4121
  }
H
hzcheng 已提交
4122

4123
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
4124 4125
  if (answerPrompt && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
4126 4127 4128 4129
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
4130
    } else {
4131
      g_args.answer_yes = false;
H
hzcheng 已提交
4132
    }
4133 4134 4135
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
4136
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
4137
    goto PARSE_OVER;
4138
  }
4139

4140 4141
  cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
  if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
4142
    if (gQueryTimes->valueint <= 0) {
4143 4144
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4145 4146
      goto PARSE_OVER;
    }
4147 4148 4149 4150
    g_args.query_times = gQueryTimes->valueint;
  } else if (!gQueryTimes) {
    g_args.query_times = 1;
  } else {
4151 4152
    errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4153 4154 4155
    goto PARSE_OVER;
  }

4156 4157
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
4158
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE);
4159
  } else if (!dbs) {
4160
    printf("ERROR: failed to read json, databases not found\n");
4161
    goto PARSE_OVER;
H
hzcheng 已提交
4162
  }
4163 4164 4165

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
4166
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
4167
  } else if (!queryMode) {
4168
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
4169
  } else {
4170
    printf("ERROR: failed to read json, query_mode not found\n");
4171
    goto PARSE_OVER;
H
hzcheng 已提交
4172
  }
4173

4174
  // specified_table_query
4175 4176
  cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!specifiedQuery) {
4177
    g_queryInfo.specifiedQueryInfo.concurrent = 1;
4178 4179
    g_queryInfo.specifiedQueryInfo.sqlCount = 0;
  } else if (specifiedQuery->type != cJSON_Object) {
4180
    printf("ERROR: failed to read json, super_table_query not found\n");
4181
    goto PARSE_OVER;
4182
  } else {
4183 4184 4185 4186 4187
    cJSON* queryInterval = cJSON_GetObjectItem(specifiedQuery, "query_interval");
    if (queryInterval && queryInterval->type == cJSON_Number) {
      g_queryInfo.specifiedQueryInfo.queryInterval = queryInterval->valueint;
    } else if (!queryInterval) {
      g_queryInfo.specifiedQueryInfo.queryInterval = 0;
4188 4189
    }

4190 4191
    cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery,
        "query_times");
4192
    if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
4193
      if (specifiedQueryTimes->valueint <= 0) {
4194 4195
        errorPrint(
                "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4196
          __func__, __LINE__, specifiedQueryTimes->valueint);
4197 4198 4199
        goto PARSE_OVER;

      }
4200 4201 4202 4203
      g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint;
    } else if (!specifiedQueryTimes) {
      g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times;
    } else {
4204 4205
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
          __func__, __LINE__);
4206
      goto PARSE_OVER;
4207
    }
4208

4209
    cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent");
4210
    if (concurrent && concurrent->type == cJSON_Number) {
4211
      if (concurrent->valueint <= 0) {
4212
        errorPrint(
4213
                "%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n",
4214 4215
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount,
4216 4217 4218
              g_queryInfo.specifiedQueryInfo.concurrent);
        goto PARSE_OVER;
      }
4219
      g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint;
4220
    } else if (!concurrent) {
4221
      g_queryInfo.specifiedQueryInfo.concurrent = 1;
4222
    }
4223

4224 4225 4226 4227 4228 4229 4230
    cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode");
    if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String
        && specifiedAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE;
4231
      } else {
4232
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4233
            __func__, __LINE__);
4234 4235 4236
        goto PARSE_OVER;
      }
    } else {
4237
      g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
4238
    }
4239

4240
    cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval");
4241
    if (interval && interval->type == cJSON_Number) {
4242
      g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint;
4243
    } else if (!interval) {
4244 4245
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4246
      g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000;
4247
    }
4248

4249
    cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart");
4250
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
4251
      if (0 == strcmp("yes", restart->valuestring)) {
4252
        g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4253
      } else if (0 == strcmp("no", restart->valuestring)) {
4254
        g_queryInfo.specifiedQueryInfo.subscribeRestart = false;
4255
      } else {
4256
        printf("ERROR: failed to read json, subscribe restart error\n");
4257 4258 4259
        goto PARSE_OVER;
      }
    } else {
4260
      g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4261
    }
4262

4263
    cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress");
4264 4265 4266
    if (keepProgress
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
4267
      if (0 == strcmp("yes", keepProgress->valuestring)) {
4268
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1;
4269
      } else if (0 == strcmp("no", keepProgress->valuestring)) {
4270
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4271
      } else {
4272
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4273 4274 4275
        goto PARSE_OVER;
      }
    } else {
4276
      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4277 4278
    }

4279
    // sqls
4280 4281
    cJSON* specifiedSqls = cJSON_GetObjectItem(specifiedQuery, "sqls");
    if (!specifiedSqls) {
4282
      g_queryInfo.specifiedQueryInfo.sqlCount = 0;
4283
    } else if (specifiedSqls->type != cJSON_Array) {
4284 4285
      errorPrint("%s() LN%d, failed to read json, super sqls not found\n",
          __func__, __LINE__);
4286
      goto PARSE_OVER;
4287
    } else {
4288 4289 4290 4291 4292 4293 4294 4295
      int superSqlSize = cJSON_GetArraySize(specifiedSqls);
      if (superSqlSize * g_queryInfo.specifiedQueryInfo.concurrent
              > MAX_QUERY_SQL_COUNT) {
        errorPrint("%s() LN%d, failed to read json, query sql(%d) * concurrent(%d) overflow, max is %d\n",
           __func__, __LINE__,
           superSqlSize,
           g_queryInfo.specifiedQueryInfo.concurrent,
           MAX_QUERY_SQL_COUNT);
4296 4297
        goto PARSE_OVER;
      }
4298

4299
      g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize;
4300
      for (int j = 0; j < superSqlSize; ++j) {
4301
        cJSON* sql = cJSON_GetArrayItem(specifiedSqls, j);
4302
        if (sql == NULL) continue;
4303

4304 4305
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
4306
          printf("ERROR: failed to read json, sql not found\n");
4307 4308
          goto PARSE_OVER;
        }
4309 4310 4311
        tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j],
                sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);

4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322
        cJSON* endAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "endAfterConsume");
        if (endAfterConsume
                && endAfterConsume->type == cJSON_Number) {
            g_queryInfo.specifiedQueryInfo.endAfterConsume[j]
                = endAfterConsume->valueint;
        } else if (!endAfterConsume) {
            // default value is -1, which mean infinite loop
            g_queryInfo.specifiedQueryInfo.endAfterConsume[j] = -1;
        }

4323 4324 4325 4326 4327 4328 4329
        cJSON* resubAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume");
        if (resubAfterConsume
                && resubAfterConsume->type == cJSON_Number) {
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j]
                = resubAfterConsume->valueint;
        } else if (!resubAfterConsume) {
4330 4331
            // default value is -1, which mean do not resub
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = -1;
4332
        }
H
hzcheng 已提交
4333

4334
        cJSON *result = cJSON_GetObjectItem(sql, "result");
4335 4336 4337 4338
        if ((NULL != result) && (result->type == cJSON_String)
                && (result->valuestring != NULL)) {
          tstrncpy(g_queryInfo.specifiedQueryInfo.result[j],
                  result->valuestring, MAX_FILE_NAME_LEN);
4339
        } else if (NULL == result) {
4340 4341
          memset(g_queryInfo.specifiedQueryInfo.result[j],
                  0, MAX_FILE_NAME_LEN);
4342
        } else {
4343
          printf("ERROR: failed to read json, super query result file not found\n");
4344
          goto PARSE_OVER;
4345
        }
4346 4347 4348
      }
    }
  }
4349

4350
  // super_table_query
4351 4352
  cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!superQuery) {
4353
    g_queryInfo.superQueryInfo.threadCnt = 1;
4354 4355
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
4356
    printf("ERROR: failed to read json, sub_table_query not found\n");
4357 4358
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
4359
  } else {
4360
    cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval");
4361
    if (subrate && subrate->type == cJSON_Number) {
4362
      g_queryInfo.superQueryInfo.queryInterval = subrate->valueint;
4363
    } else if (!subrate) {
4364
      g_queryInfo.superQueryInfo.queryInterval = 0;
4365 4366 4367 4368
    }

    cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
    if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
4369 4370 4371
      if (superQueryTimes->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
          __func__, __LINE__, superQueryTimes->valueint);
4372 4373
        goto PARSE_OVER;
      }
4374 4375 4376 4377
      g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
    } else if (!superQueryTimes) {
      g_queryInfo.superQueryInfo.queryTimes = g_args.query_times;
    } else {
4378 4379
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
          __func__, __LINE__);
4380
      goto PARSE_OVER;
4381
    }
4382

4383
    cJSON* threads = cJSON_GetObjectItem(superQuery, "threads");
4384
    if (threads && threads->type == cJSON_Number) {
4385 4386 4387 4388 4389 4390
      if (threads->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, threads input mistake\n",
          __func__, __LINE__);
        goto PARSE_OVER;

      }
4391
      g_queryInfo.superQueryInfo.threadCnt = threads->valueint;
4392
    } else if (!threads) {
4393
      g_queryInfo.superQueryInfo.threadCnt = 1;
4394
    }
4395

4396
    //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count");
4397
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
4398
    //  g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint;
4399
    //} else if (!subTblCnt) {
4400
    //  g_queryInfo.superQueryInfo.childTblCount = 0;
4401
    //}
4402

4403
    cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname");
4404 4405 4406 4407
    if (stblname && stblname->type == cJSON_String
        && stblname->valuestring != NULL) {
      tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring,
          MAX_TB_NAME_SIZE);
4408
    } else {
4409 4410
      errorPrint("%s() LN%d, failed to read json, super table name input error\n",
          __func__, __LINE__);
4411 4412
      goto PARSE_OVER;
    }
4413

4414 4415 4416 4417 4418 4419 4420
    cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode");
    if (superAsyncMode && superAsyncMode->type == cJSON_String
        && superAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE;
4421
      } else {
4422
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4423
            __func__, __LINE__);
4424 4425 4426
        goto PARSE_OVER;
      }
    } else {
4427
      g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
4428
    }
4429

4430 4431 4432 4433 4434 4435 4436 4437 4438
    cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval");
    if (superInterval && superInterval->type == cJSON_Number) {
      if (superInterval->valueint < 0) {
        errorPrint("%s() LN%d, failed to read json, interval input mistake\n",
            __func__, __LINE__);
        goto PARSE_OVER;
      }
      g_queryInfo.superQueryInfo.subscribeInterval = superInterval->valueint;
    } else if (!superInterval) {
4439 4440
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4441
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
4442
    }
4443

4444
    cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart");
4445 4446
    if (subrestart && subrestart->type == cJSON_String
        && subrestart->valuestring != NULL) {
4447
      if (0 == strcmp("yes", subrestart->valuestring)) {
4448
        g_queryInfo.superQueryInfo.subscribeRestart = true;
4449
      } else if (0 == strcmp("no", subrestart->valuestring)) {
4450
        g_queryInfo.superQueryInfo.subscribeRestart = false;
4451
      } else {
4452
        printf("ERROR: failed to read json, subscribe restart error\n");
4453 4454 4455
        goto PARSE_OVER;
      }
    } else {
4456
      g_queryInfo.superQueryInfo.subscribeRestart = true;
4457
    }
4458

4459 4460 4461 4462 4463
    cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
    if (superkeepProgress &&
            superkeepProgress->type == cJSON_String
            && superkeepProgress->valuestring != NULL) {
      if (0 == strcmp("yes", superkeepProgress->valuestring)) {
4464
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
4465
      } else if (0 == strcmp("no", superkeepProgress->valuestring)) {
4466
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4467
      } else {
4468
        printf("ERROR: failed to read json, subscribe super table keepProgress error\n");
4469 4470 4471
        goto PARSE_OVER;
      }
    } else {
4472
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4473
    }
4474

4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485
    cJSON* superEndAfterConsume =
            cJSON_GetObjectItem(superQuery, "endAfterConsume");
    if (superEndAfterConsume
            && superEndAfterConsume->type == cJSON_Number) {
        g_queryInfo.superQueryInfo.endAfterConsume =
            superEndAfterConsume->valueint;
    } else if (!superEndAfterConsume) {
        // default value is -1, which mean do not resub
        g_queryInfo.superQueryInfo.endAfterConsume = -1;
    }

4486
    cJSON* superResubAfterConsume =
4487
            cJSON_GetObjectItem(superQuery, "endAfterConsume");
4488 4489
    if (superResubAfterConsume
            && superResubAfterConsume->type == cJSON_Number) {
4490
        g_queryInfo.superQueryInfo.endAfterConsume =
4491 4492
            superResubAfterConsume->valueint;
    } else if (!superResubAfterConsume) {
4493 4494
        // default value is -1, which mean do not resub
        g_queryInfo.superQueryInfo.endAfterConsume = -1;
4495 4496
    }

4497 4498 4499
    // supert table sqls
    cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls");
    if (!superSqls) {
4500
      g_queryInfo.superQueryInfo.sqlCount = 0;
4501
    } else if (superSqls->type != cJSON_Array) {
4502 4503
      errorPrint("%s() LN%d: failed to read json, super sqls not found\n",
          __func__, __LINE__);
4504
      goto PARSE_OVER;
4505
    } else {
4506
      int superSqlSize = cJSON_GetArraySize(superSqls);
4507
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4508 4509
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4510 4511
        goto PARSE_OVER;
      }
4512

4513
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
4514
      for (int j = 0; j < superSqlSize; ++j) {
4515
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
4516
        if (sql == NULL) continue;
4517

4518
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
4519 4520 4521 4522
        if (!sqlStr || sqlStr->type != cJSON_String
            || sqlStr->valuestring == NULL) {
          errorPrint("%s() LN%d, failed to read json, sql not found\n",
              __func__, __LINE__);
4523 4524
          goto PARSE_OVER;
        }
4525 4526
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring,
            MAX_QUERY_SQL_LENGTH);
4527 4528

        cJSON *result = cJSON_GetObjectItem(sql, "result");
4529 4530 4531 4532
        if (result != NULL && result->type == cJSON_String
            && result->valuestring != NULL){
          tstrncpy(g_queryInfo.superQueryInfo.result[j],
              result->valuestring, MAX_FILE_NAME_LEN);
4533
        } else if (NULL == result) {
4534
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4535
        }  else {
4536 4537
          errorPrint("%s() LN%d, failed to read json, sub query result file not found\n",
              __func__, __LINE__);
4538
          goto PARSE_OVER;
4539
        }
4540 4541
      }
    }
H
hzcheng 已提交
4542 4543
  }

4544
  ret = true;
H
hzcheng 已提交
4545

4546 4547 4548
PARSE_OVER:
  return ret;
}
H
hzcheng 已提交
4549

4550
static bool getInfoFromJsonFile(char* file) {
4551
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
4552

4553 4554 4555 4556 4557
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
4558

4559
  bool  ret = false;
4560
  int   maxLen = 6400000;
4561 4562 4563 4564 4565 4566 4567
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
4568
  }
H
Hui Li 已提交
4569

4570 4571 4572
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
4573
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
4574 4575
    goto PARSE_OVER;
  }
H
Hui Li 已提交
4576

4577 4578 4579
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
4580
      g_args.test_mode = INSERT_TEST;
4581
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
4582
      g_args.test_mode = QUERY_TEST;
4583
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
4584
      g_args.test_mode = SUBSCRIBE_TEST;
4585
    } else {
4586
      printf("ERROR: failed to read json, filetype not support\n");
4587 4588 4589
      goto PARSE_OVER;
    }
  } else if (!filetype) {
4590
    g_args.test_mode = INSERT_TEST;
4591
  } else {
4592
    printf("ERROR: failed to read json, filetype not found\n");
4593 4594
    goto PARSE_OVER;
  }
H
hzcheng 已提交
4595

4596
  if (INSERT_TEST == g_args.test_mode) {
4597
    ret = getMetaFromInsertJsonFile(root);
4598 4599
  } else if ((QUERY_TEST == g_args.test_mode)
          || (SUBSCRIBE_TEST == g_args.test_mode)) {
4600 4601
    ret = getMetaFromQueryJsonFile(root);
  } else {
4602 4603
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n",
            __func__, __LINE__);
4604
    goto PARSE_OVER;
4605
  }
4606 4607 4608 4609

PARSE_OVER:
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
4610
  fclose(fp);
4611 4612
  return ret;
}
H
hzcheng 已提交
4613

4614
static void prepareSampleData() {
4615
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4616 4617 4618
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
        (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]);
S
Shuaiqiang Chang 已提交
4619
      }
4620 4621 4622 4623
    }
  }
}

4624
static void postFreeResource() {
4625
  tmfclose(g_fpOfInsertResult);
4626
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4627
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
4628 4629 4630
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
4631
      }
4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4644
    }
4645 4646
  }
}
S
Shuaiqiang Chang 已提交
4647

4648 4649 4650
static int getRowDataFromSample(
        char* dataBuf, int64_t maxLen, int64_t timestamp,
      SSuperTable* superTblInfo, int64_t* sampleUsePos) {
4651
  if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
4652
/*    int ret = readSampleFromCsvFileToMem(superTblInfo);
4653
    if (0 != ret) {
4654
      tmfree(superTblInfo->sampleDataBuf);
4655
      superTblInfo->sampleDataBuf = NULL;
4656 4657
      return -1;
    }
4658
*/
4659 4660
    *sampleUsePos = 0;
  }
S
Shuaiqiang Chang 已提交
4661

4662
  int    dataLen = 0;
4663

4664 4665 4666 4667
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "(%" PRId64 ", ", timestamp);
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
4668
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
S
Shuaiqiang Chang 已提交
4669

4670
  (*sampleUsePos)++;
4671

4672 4673
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4674

4675 4676
static int64_t generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stbInfo) {
  int64_t   dataLen = 0;
4677
  char  *pstr = recBuf;
4678
  int64_t maxLen = MAX_DATA_SIZE;
4679

4680
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "(%" PRId64 ",", timestamp);
4681

4682
  for (int i = 0; i < stbInfo->columnCount; i++) {
4683 4684
    if ((0 == strncasecmp(stbInfo->columns[i].dataType, "BINARY", strlen("BINARY")))
            || (0 == strncasecmp(stbInfo->columns[i].dataType, "NCHAR", strlen("NCHAR")))) {
4685
      if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
4686
        errorPrint( "binary or nchar length overflow, max size:%u\n",
4687
                (uint32_t)TSDB_MAX_BINARY_LEN);
4688
        return -1;
4689
      }
4690

4691 4692
      char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
      if (NULL == buf) {
4693
        errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
4694
        return -1;
4695 4696
      }
      rand_string(buf, stbInfo->columns[i].dataLen);
4697
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf);
4698
      tmfree(buf);
4699
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4700
                "INT", 3)) {
4701
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4702
              "%d,", rand_int());
4703
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4704
                "BIGINT", 6)) {
4705
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4706
              "%"PRId64",", rand_bigint());
4707
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4708
                "FLOAT", 5)) {
4709
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4710
              "%f,", rand_float());
4711
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4712
                "DOUBLE", 6)) {
4713
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4714
              "%f,", rand_double());
4715
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4716
                "SMALLINT", 8)) {
4717
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4718
          "%d,", rand_smallint());
4719
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4720
          "TINYINT", strlen("TINYINT"))) {
4721
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4722
          "%d,", rand_tinyint());
4723
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4724
          "BOOL", strlen("BOOL"))) {
4725
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4726
          "%d,", rand_bool());
4727
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4728
          "TIMESTAMP", strlen("TIMESTAMP"))) {
4729
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4730
          "%"PRId64",", rand_bigint());
4731
    }  else {
4732
      errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType);
4733
      return -1;
4734
    }
S
Shuaiqiang Chang 已提交
4735
  }
4736

4737
  dataLen -= 1;
4738
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")");
4739

4740 4741 4742
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

  return strlen(recBuf);
4743
}
S
Shuaiqiang Chang 已提交
4744

4745
static int64_t generateData(char *recBuf, char **data_type,
4746
        int num_of_cols, int64_t timestamp, int lenOfBinary) {
4747 4748
  memset(recBuf, 0, MAX_DATA_SIZE);
  char *pstr = recBuf;
4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762
  pstr += sprintf(pstr, "(%" PRId64, timestamp);
  int c = 0;

  for (; c < MAX_NUM_DATATYPE; c++) {
    if (data_type[c] == NULL) {
      break;
    }
  }

  if (0 == c) {
    perror("data type error!");
    exit(-1);
  }

4763
  for (int i = 0; i < c; i++) {
4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774
    if (strcasecmp(data_type[i % c], "TINYINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_tinyint() );
    } else if (strcasecmp(data_type[i % c], "SMALLINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_smallint());
    } else if (strcasecmp(data_type[i % c], "INT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_int());
    } else if (strcasecmp(data_type[i % c], "BIGINT") == 0) {
      pstr += sprintf(pstr, ",%" PRId64, rand_bigint());
    } else if (strcasecmp(data_type[i % c], "FLOAT") == 0) {
      pstr += sprintf(pstr, ",%10.4f", rand_float());
    } else if (strcasecmp(data_type[i % c], "DOUBLE") == 0) {
4775
      double t = rand_double();
4776 4777
      pstr += sprintf(pstr, ",%20.8f", t);
    } else if (strcasecmp(data_type[i % c], "BOOL") == 0) {
4778
      bool b = rand_bool() & 1;
4779 4780
      pstr += sprintf(pstr, ",%s", b ? "true" : "false");
    } else if (strcasecmp(data_type[i % c], "BINARY") == 0) {
4781 4782
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4783
      pstr += sprintf(pstr, ",\"%s\"", s);
4784
      free(s);
4785
    } else if (strcasecmp(data_type[i % c], "NCHAR") == 0) {
4786 4787
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4788
      pstr += sprintf(pstr, ",\"%s\"", s);
4789 4790 4791
      free(s);
    }

4792
    if (strlen(recBuf) > MAX_DATA_SIZE) {
4793 4794 4795 4796 4797 4798 4799
      perror("column length too long, abort");
      exit(-1);
    }
  }

  pstr += sprintf(pstr, ")");

4800 4801
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

4802
  return (int32_t)strlen(recBuf);
4803 4804
}

4805
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
4806 4807
  char* sampleDataBuf = NULL;

4808
  sampleDataBuf = calloc(
4809
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
4810
  if (sampleDataBuf == NULL) {
4811
      errorPrint("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n",
4812
              __func__, __LINE__,
4813
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE,
4814
              strerror(errno));
4815
      return -1;
4816
  }
4817

4818 4819 4820 4821
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
4822 4823
      errorPrint("%s() LN%d, read sample from csv file failed.\n",
          __func__, __LINE__);
4824
      tmfree(sampleDataBuf);
4825
      superTblInfo->sampleDataBuf = NULL;
4826
      return -1;
H
Hui Li 已提交
4827
  }
4828

4829 4830 4831
  return 0;
}

4832
static int64_t execInsert(threadInfo *pThreadInfo, char *buffer, uint64_t k)
4833 4834
{
  int affectedRows;
4835
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4836

4837 4838
  verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
            __func__, __LINE__, buffer);
4839
  if (superTblInfo) {
4840
    if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) {
4841
      affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false);
4842
    } else if (0 == strncasecmp(superTblInfo->insertMode, "rest", strlen("rest"))) {
4843 4844
      if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port,
                  buffer, NULL /* not set result file */)) {
4845
        affectedRows = -1;
4846 4847
        printf("========restful return fail, threadID[%d]\n",
            pThreadInfo->threadID);
4848 4849 4850
      } else {
        affectedRows = k;
      }
4851 4852 4853 4854
    } else {
      errorPrint("%s() LN%d: unknown insert mode: %s\n",
        __func__, __LINE__, superTblInfo->insertMode);
      affectedRows = 0;
4855
    }
4856
  } else {
4857
    affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false);
4858 4859 4860 4861 4862
  }

  return affectedRows;
}

4863 4864
static void getTableName(char *pTblName,
        threadInfo* pThreadInfo, uint64_t tableSeq)
4865 4866
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4867 4868
  if ((superTblInfo)
          && (AUTO_CREATE_SUBTBL != superTblInfo->autoCreateTable)) {
4869
    if (superTblInfo->childTblLimit > 0) {
4870
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
4871 4872
            superTblInfo->childTblName +
            (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
4873
    } else {
4874

4875
        verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n",
4876 4877
                pThreadInfo->threadID, __func__, __LINE__,
                pThreadInfo->start_table_from,
4878 4879 4880 4881 4882
                pThreadInfo->ntables, tableSeq);
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
    }
  } else {
4883
    snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"",
4884
        g_args.tb_prefix, tableSeq);
4885 4886 4887
  }
}

4888
static int64_t generateDataTail(
4889
        SSuperTable* superTblInfo,
4890
        uint64_t batch, char* buffer, int64_t remainderBufLen, int64_t insertRows,
4891
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) {
4892 4893
  uint64_t len = 0;
  uint32_t ncols_per_record = 1; // count first col ts
4894

4895 4896
  char *pstr = buffer;

4897
  if (superTblInfo == NULL) {
4898
    uint32_t datatypeSeq = 0;
4899 4900 4901 4902
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
4903 4904
  }

4905
  verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch);
4906

4907 4908 4909 4910 4911 4912 4913 4914
  bool tsRand;
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
                  "rand", strlen("rand")))) {
        tsRand = true;
    } else {
        tsRand = false;
  }

4915
  uint64_t k = 0;
4916
  for (k = 0; k < batch;) {
4917
    char data[MAX_DATA_SIZE];
4918 4919
    memset(data, 0, MAX_DATA_SIZE);

4920
    int64_t retLen = 0;
4921

4922
    if (superTblInfo) {
4923 4924 4925 4926 4927 4928 4929 4930 4931
        if (tsRand) {
            retLen = generateRowData(
                    data,
                    startTime + getTSRandTail(
                        superTblInfo->timeStampStep, k,
                        superTblInfo->disorderRatio,
                        superTblInfo->disorderRange),
                    superTblInfo);
        } else {
4932
          retLen = getRowDataFromSample(
4933 4934
                    data,
                    remainderBufLen,
4935
                    startTime + superTblInfo->timeStampStep * k,
4936
                    superTblInfo,
4937
                    pSamplePos);
4938
        }
4939 4940
        if (retLen > remainderBufLen) {
            break;
4941
        }
4942

4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958
        pstr += snprintf(pstr , retLen + 1, "%s", data);
        k++;
        len += retLen;
        remainderBufLen -= retLen;
    } else {
        char **data_type = g_args.datatype;
        int lenOfBinary = g_args.len_of_binary;
        retLen = generateData(data, data_type,
                ncols_per_record,
                startTime + getTSRandTail(
                    DEFAULT_TIMESTAMP_STEP, k,
                    g_args.disorderRatio,
                    g_args.disorderRange),
                lenOfBinary);
        if (len > remainderBufLen)
            break;
4959

4960 4961 4962 4963
        pstr += sprintf(pstr, "%s", data);
        k++;
        len += retLen;
        remainderBufLen -= retLen;
4964 4965
    }

4966
    verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n",
4967
            __func__, __LINE__, len, k, buffer);
4968 4969

    startFrom ++;
4970

4971
    if (startFrom >= insertRows) {
4972
      break;
4973
    }
4974
  }
4975

4976 4977 4978
  *dataLen = len;
  return k;
}
4979

4980
static int generateSQLHead(char *tableName, int32_t tableSeq,
4981 4982
        threadInfo* pThreadInfo, SSuperTable* superTblInfo,
        char *buffer, int remainderBufLen)
4983 4984
{
  int len;
4985 4986 4987 4988

#define HEAD_BUFF_LEN    1024*24  // 16*1024 + (192+32)*2 + insert into ..
  char headBuf[HEAD_BUFF_LEN];

4989
  if (superTblInfo) {
4990 4991
    if ((AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable)
            && (TBL_ALREADY_EXISTS != superTblInfo->childTblExists)) {
4992 4993
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
4994
            tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq);
4995 4996 4997 4998 4999 5000
      } else {
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
      }
      if (NULL == tagsValBuf) {
5001 5002
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n",
            __func__, __LINE__);
5003 5004 5005
        return -1;
      }

5006 5007 5008
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
5009
                  "%s.%s using %s.%s tags %s values",
5010 5011 5012 5013 5014 5015 5016
                  pThreadInfo->db_name,
                  tableName,
                  pThreadInfo->db_name,
                  superTblInfo->sTblName,
                  tagsValBuf);
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
5017 5018 5019
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
5020
                  "%s.%s values",
5021
                  pThreadInfo->db_name,
5022
                  tableName);
5023
    } else {
5024 5025 5026
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
5027
                  "%s.%s values",
5028 5029 5030 5031
                  pThreadInfo->db_name,
                  tableName);
    }
  } else {
5032 5033 5034
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
5035
                  "%s.%s values",
5036 5037 5038 5039
                  pThreadInfo->db_name,
                  tableName);
  }

5040 5041 5042 5043 5044
  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

5045 5046 5047
  return len;
}

5048
static int64_t generateInterlaceDataBuffer(
5049 5050
        char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes,
        uint64_t tableSeq,
5051
        threadInfo *pThreadInfo, char *buffer,
5052
        int64_t insertRows,
5053
        int64_t startTime,
5054
        uint64_t *pRemainderBufLen)
5055
{
5056
  assert(buffer);
5057 5058 5059 5060 5061 5062 5063 5064 5065 5066
  char *pstr = buffer;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo,
            superTblInfo, pstr, *pRemainderBufLen);

  if (headLen <= 0) {
    return 0;
  }
  // generate data buffer
5067
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n",
5068 5069 5070 5071 5072
            pThreadInfo->threadID, __func__, __LINE__, i, buffer);

  pstr += headLen;
  *pRemainderBufLen -= headLen;

5073
  int64_t dataLen = 0;
5074

5075
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%"PRIu64" batchPerTbl = %"PRIu64"\n",
5076 5077 5078 5079 5080 5081 5082 5083 5084 5085
            pThreadInfo->threadID, __func__, __LINE__,
            i, batchPerTblTimes, batchPerTbl);

  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
      startTime = taosGetTimestamp(pThreadInfo->time_precision);
    }
  } else {
      startTime = 1500000000000;
  }
5086

5087
  int64_t k = generateDataTail(
5088
    superTblInfo,
5089 5090 5091 5092
    batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
    startTime,
    &(pThreadInfo->samplePos), &dataLen);

5093
  if (k == batchPerTbl) {
5094 5095 5096
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
  } else {
5097
    debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n",
5098
            __func__, __LINE__, k, batchPerTbl);
5099 5100
    pstr -= headLen;
    pstr[0] = '\0';
5101
    k = 0;
5102 5103 5104 5105 5106
  }

  return k;
}

5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122
static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq,
          int disorderRatio, int disorderRange)
{
    int64_t randTail = timeStampStep * seq;
    if (disorderRatio > 0) {
        int rand_num = taosRandom() % 100;
        if(rand_num < disorderRatio) {
            randTail = (randTail +
                    (taosRandom() % disorderRange + 1)) * (-1);
            debugPrint("rand data generated, back %"PRId64"\n", randTail);
        }
    }

    return randTail;
}

5123
static int64_t generateProgressiveDataBuffer(
5124
        char *tableName,
5125
        int64_t tableSeq,
5126 5127
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
5128
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos,
5129
        int64_t *pRemainderBufLen)
5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  int ncols_per_record = 1; // count first col ts

  if (superTblInfo == NULL) {
    int datatypeSeq = 0;
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
  }

  assert(buffer != NULL);
5144
  char *pstr = buffer;
5145

5146
  int64_t k = 0;
5147

5148
  memset(buffer, 0, *pRemainderBufLen);
5149

5150 5151
  int64_t headLen = generateSQLHead(tableName, tableSeq, pThreadInfo,
          superTblInfo,
5152
          buffer, *pRemainderBufLen);
5153 5154 5155 5156

  if (headLen <= 0) {
    return 0;
  }
5157
  pstr += headLen;
5158
  *pRemainderBufLen -= headLen;
5159

5160
  int64_t dataLen;
5161
  k = generateDataTail(superTblInfo,
5162
          g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom,
5163
          startTime,
5164
          pSamplePos, &dataLen);
5165

5166 5167
  return k;
}
5168

5169 5170 5171 5172 5173 5174 5175 5176 5177
static void printStatPerThread(threadInfo *pThreadInfo)
{
  fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n",
          pThreadInfo->threadID,
          pThreadInfo->totalInsertRows,
          pThreadInfo->totalAffectedRows,
          (double)(pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay/1000.0)));
}

5178
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
5179 5180
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
         pThreadInfo->threadID, __func__, __LINE__);
5181

5182
  int64_t insertRows;
5183
  uint64_t interlaceRows;
5184

5185
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5186

5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199
  if (superTblInfo) {
    insertRows = superTblInfo->insertRows;

    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    insertRows = g_args.num_of_DPT;
    interlaceRows = g_args.interlace_rows;
  }
5200

5201
  if (interlaceRows > insertRows)
5202
    interlaceRows = insertRows;
5203

5204 5205
  if (interlaceRows > g_args.num_of_RPR)
    interlaceRows = g_args.num_of_RPR;
5206

5207 5208 5209 5210 5211 5212 5213 5214 5215 5216
  int insertMode;

  if (interlaceRows > 0) {
    insertMode = INTERLACE_INSERT_MODE;
  } else {
    insertMode = PROGRESSIVE_INSERT_MODE;
  }

  // TODO: prompt tbl count multple interlace rows and batch
  //
5217

5218
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
5219
  char* buffer = calloc(maxSqlLen, 1);
5220
  if (NULL == buffer) {
5221
    errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5222
              __func__, __LINE__, maxSqlLen, strerror(errno));
5223 5224 5225
    return NULL;
  }

5226
  char tableName[TSDB_TABLE_NAME_LEN];
5227 5228 5229 5230

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

5231
  int64_t nTimeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
5232

5233
  uint64_t insert_interval =
5234
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5235 5236
  uint64_t st = 0;
  uint64_t et = UINT64_MAX;
5237

5238 5239 5240
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5241

5242
  uint64_t tableSeq = pThreadInfo->start_table_from;
5243

5244
  debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n",
5245
          pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from,
5246 5247 5248 5249
          pThreadInfo->ntables, insertRows);

  int64_t startTime = pThreadInfo->start_time;

5250 5251
  uint64_t batchPerTbl = interlaceRows;
  uint64_t batchPerTblTimes;
5252

5253
  if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) {
5254
    batchPerTblTimes =
5255
        g_args.num_of_RPR / interlaceRows;
5256 5257 5258 5259
  } else {
    batchPerTblTimes = 1;
  }

5260
  uint64_t generatedRecPerTbl = 0;
5261
  bool flagSleep = true;
5262
  uint64_t sleepTimeTotal = 0;
5263

5264 5265 5266
  char *strInsertInto = "insert into ";
  int nInsertBufLen = strlen(strInsertInto);

5267
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
5268
    if ((flagSleep) && (insert_interval)) {
5269
        st = taosGetTimestampMs();
5270
        flagSleep = false;
5271 5272
    }
    // generate data
5273
    memset(buffer, 0, maxSqlLen);
5274
    uint64_t remainderBufLen = maxSqlLen;
5275

5276
    char *pstr = buffer;
5277 5278 5279 5280 5281

    int len = snprintf(pstr, nInsertBufLen + 1, "%s", strInsertInto);
    pstr += len;
    remainderBufLen -= len;

5282
    uint64_t recOfBatch = 0;
5283

5284
    for (uint64_t i = 0; i < batchPerTblTimes; i ++) {
5285
      getTableName(tableName, pThreadInfo, tableSeq);
5286 5287 5288
      if (0 == strlen(tableName)) {
        errorPrint("[%d] %s() LN%d, getTableName return null\n",
            pThreadInfo->threadID, __func__, __LINE__);
5289
        free(buffer);
5290 5291
        return NULL;
      }
5292

5293
      uint64_t oldRemainderLen = remainderBufLen;
5294
      int64_t generated = generateInterlaceDataBuffer(
5295 5296 5297 5298
        tableName, batchPerTbl, i, batchPerTblTimes,
        tableSeq,
        pThreadInfo, pstr,
        insertRows,
5299
        startTime,
5300
        &remainderBufLen);
5301

5302 5303
      debugPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5304
      if (generated < 0) {
5305
        errorPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
5306
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5307
        goto free_of_interlace;
5308 5309
      } else if (generated == 0) {
        break;
5310 5311
      }

5312
      tableSeq ++;
5313
      recOfBatch += batchPerTbl;
5314
      pstr += (oldRemainderLen - remainderBufLen);
5315
//      startTime += batchPerTbl * superTblInfo->timeStampStep;
5316
      pThreadInfo->totalInsertRows += batchPerTbl;
5317
      verbosePrint("[%d] %s() LN%d batchPerTbl=%"PRId64" recOfBatch=%"PRId64"\n",
5318 5319
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
5320

5321 5322 5323 5324
      if (insertMode == INTERLACE_INSERT_MODE) {
          if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
            // turn to first table
            tableSeq = pThreadInfo->start_table_from;
5325
            generatedRecPerTbl += batchPerTbl;
5326 5327

            startTime = pThreadInfo->start_time
5328
              + generatedRecPerTbl * nTimeStampStep;
5329

5330 5331 5332 5333
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;

5334 5335 5336 5337
            int remainRows = insertRows - generatedRecPerTbl;
            if ((remainRows > 0) && (batchPerTbl > remainRows))
              batchPerTbl = remainRows;

5338 5339
            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
5340
          }
5341 5342
      }

5343
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n",
5344 5345 5346 5347
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
5348 5349 5350
        break;
    }

5351
    verbosePrint("[%d] %s() LN%d recOfBatch=%"PRIu64" totalInsertRows=%"PRIu64"\n",
5352 5353 5354 5355 5356
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
           pThreadInfo->threadID, __func__, __LINE__, buffer);

5357
    startTs = taosGetTimestampMs();
5358

5359 5360 5361 5362 5363 5364 5365
    if (recOfBatch == 0) {
      errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n",
              pThreadInfo->threadID, __func__, __LINE__,
              recOfBatch);
      errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n");
      goto free_of_interlace;
    }
5366
    int64_t affectedRows = execInsert(pThreadInfo, buffer, recOfBatch);
5367

5368
    endTs = taosGetTimestampMs();
5369 5370
    uint64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n",
5371
            __func__, __LINE__, delay);
5372 5373 5374
    verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5375 5376 5377 5378 5379 5380

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

5381 5382
    if (recOfBatch != affectedRows) {
        errorPrint("[%d] %s() LN%d execInsert insert %"PRIu64", affected rows: %"PRId64"\n%s\n",
5383
                pThreadInfo->threadID, __func__, __LINE__,
5384
                recOfBatch, affectedRows, buffer);
5385
        goto free_of_interlace;
5386
    }
5387

5388
    pThreadInfo->totalAffectedRows += affectedRows;
5389

5390 5391
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
5392
      printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n",
5393 5394 5395
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5396 5397
      lastPrintTime = currentPrintTime;
    }
5398

5399
    if ((insert_interval) && flagSleep) {
5400
      et = taosGetTimestampMs();
5401

5402 5403
      if (insert_interval > (et - st) ) {
        int sleepTime = insert_interval - (et -st);
5404 5405
        performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleepTime);
5406 5407 5408
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
5409
    }
5410 5411
  }

5412
free_of_interlace:
5413
  tmfree(buffer);
5414
  printStatPerThread(pThreadInfo);
5415 5416 5417
  return NULL;
}

5418 5419 5420 5421 5422 5423 5424 5425
// sync insertion
/*
   1 thread: 100 tables * 2000  rows/s
   1 thread: 10  tables * 20000 rows/s
   6 thread: 300 tables * 2000  rows/s

   2 taosinsertdata , 1 thread:  10  tables * 20000 rows/s
*/
5426
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
5427
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
5428

5429
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5430
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
5431

5432
  char* buffer = calloc(maxSqlLen, 1);
5433
  if (NULL == buffer) {
5434
    errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5435
              maxSqlLen,
5436 5437 5438
              strerror(errno));
    return NULL;
  }
5439

5440 5441 5442
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5443

5444
  int64_t timeStampStep =
5445
      superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
5446
/*  int insert_interval =
5447
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5448 5449
  uint64_t st = 0;
  uint64_t et = 0xffffffff;
5450
  */
5451

5452 5453
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
5454

5455
  pThreadInfo->samplePos = 0;
5456

5457
  for (uint64_t tableSeq =
5458
          pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to;
5459
        tableSeq ++) {
5460
    int64_t start_time = pThreadInfo->start_time;
5461

5462
    int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
5463

5464 5465
    verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows);

5466
    for (uint64_t i = 0; i < insertRows;) {
5467
        /*
5468
      if (insert_interval) {
5469
            st = taosGetTimestampMs();
5470
      }
5471
      */
5472

5473 5474
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
5475
      verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n",
5476 5477 5478
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);

5479
      int64_t remainderBufLen = maxSqlLen;
5480 5481 5482 5483 5484 5485 5486 5487
      char *pstr = buffer;
      int nInsertBufLen = strlen("insert into ");

      int len = snprintf(pstr, nInsertBufLen + 1, "%s", "insert into ");

      pstr += len;
      remainderBufLen -= len;

5488
      int64_t generated = generateProgressiveDataBuffer(
5489
              tableName, tableSeq, pThreadInfo, pstr, insertRows,
5490
            i, start_time,
5491 5492
            &(pThreadInfo->samplePos),
            &remainderBufLen);
5493 5494 5495
      if (generated > 0)
        i += generated;
      else
5496
        goto free_of_progressive;
5497

5498
      start_time +=  generated * timeStampStep;
5499
      pThreadInfo->totalInsertRows += generated;
5500

5501
      startTs = taosGetTimestampMs();
5502

5503
      int64_t affectedRows = execInsert(pThreadInfo, buffer, generated);
5504

5505
      endTs = taosGetTimestampMs();
5506
      uint64_t delay = endTs - startTs;
5507 5508
      performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n",
              __func__, __LINE__, delay);
5509 5510 5511
      verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5512

5513 5514 5515 5516
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
5517

5518 5519 5520 5521 5522
      if (affectedRows < 0) {
        errorPrint("%s() LN%d, affected rows: %"PRId64"\n",
                __func__, __LINE__, affectedRows);
        goto free_of_progressive;
      }
5523 5524 5525

      pThreadInfo->totalAffectedRows += affectedRows;

5526 5527
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
5528
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
5529 5530 5531
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5532 5533 5534
        lastPrintTime = currentPrintTime;
      }

5535
      if (i >= insertRows)
5536
        break;
5537
/*
5538
      if (insert_interval) {
5539
        et = taosGetTimestampMs();
5540

5541 5542
        if (insert_interval > ((et - st)) ) {
            int sleep_time = insert_interval - (et -st);
5543 5544
            performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleep_time);
5545 5546
            taosMsleep(sleep_time); // ms
        }
5547
      }
5548
      */
5549 5550
    }   // num_of_DPT

5551 5552
    if (g_args.verbose_print) {
      if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo &&
5553 5554
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
5555
          verbosePrint("%s() LN%d samplePos=%"PRId64"\n",
5556
                  __func__, __LINE__, pThreadInfo->samplePos);
5557
      }
5558
    }
5559
  } // tableSeq
5560

5561
free_of_progressive:
5562
  tmfree(buffer);
5563
  printStatPerThread(pThreadInfo);
5564
  return NULL;
H
Hui Li 已提交
5565 5566
}

5567 5568
static void* syncWrite(void *sarg) {

5569 5570
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5571

5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583
  int interlaceRows;

  if (superTblInfo) {
    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    interlaceRows = g_args.interlace_rows;
  }
5584

5585
  if (interlaceRows > 0) {
5586
    // interlace mode
5587
    return syncWriteInterlace(pThreadInfo);
5588 5589
  } else {
    // progressive mode
5590
    return syncWriteProgressive(pThreadInfo);
5591
  }
5592

5593 5594
}

5595
static void callBack(void *param, TAOS_RES *res, int code) {
5596 5597
  threadInfo* pThreadInfo = (threadInfo*)param;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5598

5599 5600
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5601
  if (insert_interval) {
5602 5603 5604
    pThreadInfo->et = taosGetTimestampMs();
    if ((pThreadInfo->et - pThreadInfo->st) < insert_interval) {
      taosMsleep(insert_interval - (pThreadInfo->et - pThreadInfo->st)); // ms
5605
    }
H
Hui Li 已提交
5606
  }
5607

5608
  char *buffer = calloc(1, pThreadInfo->superTblInfo->maxSqlLen);
5609
  char data[MAX_DATA_SIZE];
5610
  char *pstr = buffer;
5611 5612
  pstr += sprintf(pstr, "insert into %s.%s%"PRId64" values",
          pThreadInfo->db_name, pThreadInfo->tb_prefix,
5613 5614 5615 5616 5617 5618 5619 5620
          pThreadInfo->start_table_from);
//  if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
  if (pThreadInfo->counter >= g_args.num_of_RPR) {
    pThreadInfo->start_table_from++;
    pThreadInfo->counter = 0;
  }
  if (pThreadInfo->start_table_from > pThreadInfo->end_table_to) {
    tsem_post(&pThreadInfo->lock_sem);
5621 5622 5623 5624
    free(buffer);
    taos_free_result(res);
    return;
  }
5625

5626
  for (int i = 0; i < g_args.num_of_RPR; i++) {
5627
    int rand_num = taosRandom() % 100;
5628 5629
    if (0 != pThreadInfo->superTblInfo->disorderRatio
            && rand_num < pThreadInfo->superTblInfo->disorderRatio) {
5630 5631
      int64_t d = pThreadInfo->lastTs
          - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1);
5632
      generateRowData(data, d, pThreadInfo->superTblInfo);
5633
    } else {
5634
      generateRowData(data, pThreadInfo->lastTs += 1000, pThreadInfo->superTblInfo);
H
Hui Li 已提交
5635
    }
5636
    pstr += sprintf(pstr, "%s", data);
5637
    pThreadInfo->counter++;
H
Hui Li 已提交
5638

5639
    if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
5640
      break;
H
Hui Li 已提交
5641 5642
    }
  }
5643

5644
  if (insert_interval) {
5645
    pThreadInfo->st = taosGetTimestampMs();
5646
  }
5647
  taos_query_a(pThreadInfo->taos, buffer, callBack, pThreadInfo);
5648
  free(buffer);
H
Hui Li 已提交
5649

5650
  taos_free_result(res);
H
Hui Li 已提交
5651 5652
}

5653
static void *asyncWrite(void *sarg) {
5654 5655
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5656

5657 5658 5659
  pThreadInfo->st = 0;
  pThreadInfo->et = 0;
  pThreadInfo->lastTs = pThreadInfo->start_time;
5660

5661
  int insert_interval =
5662
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5663
  if (insert_interval) {
5664
    pThreadInfo->st = taosGetTimestampMs();
H
Hui Li 已提交
5665
  }
5666
  taos_query_a(pThreadInfo->taos, "show databases", callBack, pThreadInfo);
5667

5668
  tsem_wait(&(pThreadInfo->lock_sem));
H
Hui Li 已提交
5669 5670 5671 5672

  return NULL;
}

5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698
static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in *serv_addr)
{
  uint16_t rest_port = port + TSDB_PORT_HTTP;
  struct hostent *server = gethostbyname(host);
  if ((server == NULL) || (server->h_addr == NULL)) {
    errorPrint("%s", "ERROR, no such host");
    return -1;
  }

  debugPrint("h_name: %s\nh_addr=%p\nh_addretype: %s\nh_length: %d\n",
            server->h_name,
            server->h_addr,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

  memset(serv_addr, 0, sizeof(struct sockaddr_in));
  serv_addr->sin_family = AF_INET;
  serv_addr->sin_port = htons(rest_port);
#ifdef WINDOWS
  serv_addr->sin_addr.s_addr = inet_addr(host);
#else
  memcpy(&(serv_addr->sin_addr.s_addr), server->h_addr, server->h_length);
#endif
  return 0;
}

5699 5700
static void startMultiThreadInsertData(int threads, char* db_name,
        char* precision,SSuperTable* superTblInfo) {
5701

5702 5703
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  assert(pids != NULL);
H
Hui Li 已提交
5704

5705 5706
  threadInfo *infos = malloc(threads * sizeof(threadInfo));
  assert(infos != NULL);
5707

5708 5709
  memset(pids, 0, threads * sizeof(pthread_t));
  memset(infos, 0, threads * sizeof(threadInfo));
5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726

  //TAOS* taos;
  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
  //  if (NULL == taos) {
  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
  //    exit(-1);
  //  }
  //}

  int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
  if (0 != precision[0]) {
    if (0 == strncasecmp(precision, "ms", 2)) {
      timePrec = TSDB_TIME_PRECISION_MILLI;
    }  else if (0 == strncasecmp(precision, "us", 2)) {
      timePrec = TSDB_TIME_PRECISION_MICRO;
    }  else {
5727
      errorPrint("Not support precision: %s\n", precision);
5728 5729 5730 5731
      exit(-1);
    }
  }

5732
  int64_t start_time;
5733 5734 5735
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
        start_time = taosGetTimestamp(timePrec);
5736
    } else {
5737 5738 5739 5740 5741
      if (TSDB_CODE_SUCCESS != taosParseTime(
        superTblInfo->startTimestamp,
        &start_time,
        strlen(superTblInfo->startTimestamp),
        timePrec, 0)) {
5742
          ERROR_EXIT("failed to parse time!\n");
5743
      }
5744
    }
5745 5746
  } else {
     start_time = 1500000000000;
5747 5748
  }

5749
  int64_t start = taosGetTimestampMs();
5750

5751
  // read sample data from file first
5752
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5753
              "sample", strlen("sample")))) {
5754
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5755 5756
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
              __func__, __LINE__);
5757 5758 5759 5760
      exit(-1);
    }
  }

5761
  // read sample data from file first
5762
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5763 5764
              "sample", strlen("sample")))) {
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5765 5766
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
              __func__, __LINE__);
5767 5768 5769 5770 5771 5772 5773 5774
      exit(-1);
    }
  }

  TAOS* taos = taos_connect(
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
  if (NULL == taos) {
5775 5776
    errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
                __func__, __LINE__, taos_errstr(NULL));
5777 5778 5779
    exit(-1);
  }

5780
  int64_t ntables = 0;
5781
  uint64_t startFrom;
5782

5783
  if (superTblInfo) {
5784 5785
    int64_t limit;
    uint64_t offset;
5786

5787
    if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) &&
5788
            ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) {
5789
      printf("WARNING: offset and limit will not be used since the child tables not exists!\n");
5790 5791
    }

5792
    if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) {
5793 5794
      if ((superTblInfo->childTblLimit < 0)
          || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit)
5795
            > (superTblInfo->childTblCount))) {
5796 5797 5798 5799 5800 5801
        superTblInfo->childTblLimit =
            superTblInfo->childTblCount - superTblInfo->childTblOffset;
      }

      offset = superTblInfo->childTblOffset;
      limit = superTblInfo->childTblLimit;
5802
    } else {
5803 5804
      limit = superTblInfo->childTblCount;
      offset = 0;
5805 5806
    }

5807 5808 5809
    ntables = limit;
    startFrom = offset;

5810 5811 5812 5813
    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
        && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
            > superTblInfo->childTblCount)) {
      printf("WARNING: specified offset + limit > child table count!\n");
5814
      prompt();
5815 5816 5817 5818 5819
    }

    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
            && (0 == superTblInfo->childTblLimit)) {
      printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
5820
      prompt();
5821 5822
    }

5823 5824 5825
    superTblInfo->childTblName = (char*)calloc(1,
        limit * TSDB_TABLE_NAME_LEN);
    if (superTblInfo->childTblName == NULL) {
5826
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
5827 5828 5829 5830
      taos_close(taos);
      exit(-1);
    }

5831
    int64_t childTblCount;
5832 5833 5834 5835 5836 5837
    getChildNameOfSuperTableWithLimitAndOffset(
        taos,
        db_name, superTblInfo->sTblName,
        &superTblInfo->childTblName, &childTblCount,
        limit,
        offset);
5838 5839 5840
  } else {
    ntables = g_args.num_of_tables;
    startFrom = 0;
5841
  }
5842

5843 5844
  taos_close(taos);

5845
  int64_t a = ntables / threads;
5846 5847 5848 5849 5850
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

5851
  int64_t b = 0;
5852 5853 5854 5855
  if (threads != 0) {
    b = ntables % threads;
  }

5856 5857 5858 5859 5860 5861
  if ((superTblInfo)
      && (0 == strncasecmp(superTblInfo->insertMode, "rest", strlen("rest")))) {
    if (convertHostToServAddr(g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0)
      exit(-1);
  }

H
Hui Li 已提交
5862
  for (int i = 0; i < threads; i++) {
5863 5864 5865 5866 5867
    threadInfo *pThreadInfo = infos + i;
    pThreadInfo->threadID = i;
    tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE);
    pThreadInfo->time_precision = timePrec;
    pThreadInfo->superTblInfo = superTblInfo;
5868

5869 5870
    pThreadInfo->start_time = start_time;
    pThreadInfo->minDelay = UINT64_MAX;
5871

5872 5873
    if ((NULL == superTblInfo) ||
            (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) {
5874 5875
      //pThreadInfo->taos = taos;
      pThreadInfo->taos = taos_connect(
5876
              g_Dbs.host, g_Dbs.user,
5877
              g_Dbs.password, db_name, g_Dbs.port);
5878
      if (NULL == pThreadInfo->taos) {
5879 5880
        errorPrint(
                "connect to server fail from insert sub thread, reason: %s\n",
5881
                taos_errstr(NULL));
5882 5883 5884
        exit(-1);
      }
    } else {
5885
      pThreadInfo->taos = NULL;
5886 5887
    }

5888
/*    if ((NULL == superTblInfo)
5889
            || (0 == superTblInfo->multiThreadWriteOneTbl)) {
5890
            */
5891 5892 5893 5894
      pThreadInfo->start_table_from = startFrom;
      pThreadInfo->ntables = i<b?a+1:a;
      pThreadInfo->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = pThreadInfo->end_table_to + 1;
5895
/*    } else {
5896 5897 5898
      pThreadInfo->start_table_from = 0;
      pThreadInfo->ntables = superTblInfo->childTblCount;
      pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint();
5899
    }
5900
*/
5901
    tsem_init(&(pThreadInfo->lock_sem), 0, 0);
5902
    if (ASYNC_MODE == g_Dbs.asyncMode) {
5903
      pthread_create(pids + i, NULL, asyncWrite, pThreadInfo);
5904
    } else {
5905
      pthread_create(pids + i, NULL, syncWrite, pThreadInfo);
5906
    }
H
Hui Li 已提交
5907
  }
5908

H
Hui Li 已提交
5909 5910 5911 5912
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

5913 5914 5915 5916
  uint64_t totalDelay = 0;
  uint64_t maxDelay = 0;
  uint64_t minDelay = UINT64_MAX;
  uint64_t cntDelay = 1;
5917 5918
  double  avgDelay = 0;

H
Hui Li 已提交
5919
  for (int i = 0; i < threads; i++) {
5920
    threadInfo *pThreadInfo = infos + i;
5921

5922 5923
    tsem_destroy(&(pThreadInfo->lock_sem));
    taos_close(pThreadInfo->taos);
5924

5925
    debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n",
5926
            __func__, __LINE__,
5927 5928
            pThreadInfo->threadID, pThreadInfo->totalInsertRows,
            pThreadInfo->totalAffectedRows);
5929
    if (superTblInfo) {
5930 5931
        superTblInfo->totalAffectedRows += pThreadInfo->totalAffectedRows;
        superTblInfo->totalInsertRows += pThreadInfo->totalInsertRows;
5932
    } else {
5933 5934
        g_args.totalAffectedRows += pThreadInfo->totalAffectedRows;
        g_args.totalInsertRows += pThreadInfo->totalInsertRows;
5935
    }
5936

5937 5938 5939 5940
    totalDelay  += pThreadInfo->totalDelay;
    cntDelay   += pThreadInfo->cntDelay;
    if (pThreadInfo->maxDelay > maxDelay) maxDelay = pThreadInfo->maxDelay;
    if (pThreadInfo->minDelay < minDelay) minDelay = pThreadInfo->minDelay;
H
Hui Li 已提交
5941
  }
5942
  cntDelay -= 1;
H
Hui Li 已提交
5943

5944
  if (cntDelay == 0)    cntDelay = 1;
5945 5946
  avgDelay = (double)totalDelay / cntDelay;

5947 5948
  int64_t end = taosGetTimestampMs();
  int64_t t = end - start;
5949 5950

  if (superTblInfo) {
5951
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
5952
          t / 1000.0, superTblInfo->totalInsertRows,
5953 5954
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5955
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
5956 5957 5958 5959

    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
5960
          t / 1000.0, superTblInfo->totalInsertRows,
5961 5962
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5963
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
5964
    }
5965
  } else {
5966
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
5967
          t / 1000.0, g_args.totalInsertRows,
5968 5969
          g_args.totalAffectedRows,
          threads, db_name,
5970
          (double)g_args.totalInsertRows / (t / 1000.0));
5971 5972 5973
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
5974
          t * 1000.0, g_args.totalInsertRows,
5975 5976
          g_args.totalAffectedRows,
          threads, db_name,
5977
          (double)g_args.totalInsertRows / (t / 1000.0));
5978
    }
5979
  }
5980

5981
  fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
5982
          avgDelay, maxDelay, minDelay);
5983 5984
  if (g_fpOfInsertResult) {
    fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
5985
          avgDelay, maxDelay, minDelay);
5986
  }
5987

5988 5989
  //taos_close(taos);

H
Hui Li 已提交
5990
  free(pids);
5991
  free(infos);
H
Hui Li 已提交
5992 5993
}

5994
static void *readTable(void *sarg) {
5995
#if 1
5996 5997
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  TAOS *taos = pThreadInfo->taos;
H
hzcheng 已提交
5998
  char command[BUFFER_SIZE] = "\0";
5999 6000 6001
  uint64_t sTime = pThreadInfo->start_time;
  char *tb_prefix = pThreadInfo->tb_prefix;
  FILE *fp = fopen(pThreadInfo->filePath, "a");
H
Hui Li 已提交
6002
  if (NULL == fp) {
6003
    errorPrint( "fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno));
H
Hui Li 已提交
6004 6005
    return NULL;
  }
6006

6007
  int64_t num_of_DPT;
6008 6009
/*  if (pThreadInfo->superTblInfo) {
    num_of_DPT = pThreadInfo->superTblInfo->insertRows; //  nrecords_per_table;
6010
  } else {
6011
  */
6012
      num_of_DPT = g_args.num_of_DPT;
6013
//  }
6014

6015
  int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
6016
  int64_t totalData = num_of_DPT * num_of_tables;
6017
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6018 6019 6020 6021 6022

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
6023
  printf("%"PRId64" records:\n", totalData);
H
hzcheng 已提交
6024 6025
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

6026
  for (int j = 0; j < n; j++) {
H
hzcheng 已提交
6027
    double totalT = 0;
6028
    uint64_t count = 0;
6029 6030
    for (int64_t i = 0; i < num_of_tables; i++) {
      sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64,
6031
              aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
6032

6033
      double t = taosGetTimestampMs();
S
Shuaiqiang Chang 已提交
6034 6035
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
6036

S
Shuaiqiang Chang 已提交
6037
      if (code != 0) {
6038
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6039
        taos_free_result(pSql);
H
hzcheng 已提交
6040
        taos_close(taos);
6041
        fclose(fp);
6042
        return NULL;
H
hzcheng 已提交
6043 6044
      }

6045
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6046 6047 6048
        count++;
      }

6049
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6050 6051
      totalT += t;

S
Shuaiqiang Chang 已提交
6052
      taos_free_result(pSql);
H
hzcheng 已提交
6053 6054
    }

6055
    fprintf(fp, "|%10s  |   %"PRId64"   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
6056
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
6057
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
6058
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000);
H
hzcheng 已提交
6059 6060 6061
  }
  fprintf(fp, "\n");
  fclose(fp);
6062
#endif
H
hzcheng 已提交
6063 6064 6065
  return NULL;
}

6066
static void *readMetric(void *sarg) {
6067
#if 1
6068 6069
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  TAOS *taos = pThreadInfo->taos;
H
hzcheng 已提交
6070
  char command[BUFFER_SIZE] = "\0";
6071
  FILE *fp = fopen(pThreadInfo->filePath, "a");
H
Hui Li 已提交
6072
  if (NULL == fp) {
6073
    printf("fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno));
H
Hui Li 已提交
6074 6075
    return NULL;
  }
6076

6077 6078
  int64_t num_of_DPT = pThreadInfo->superTblInfo->insertRows;
  int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
6079
  int64_t totalData = num_of_DPT * num_of_tables;
6080
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6081 6082 6083 6084 6085

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
6086 6087
  printf("%"PRId64" records:\n", totalData);
  fprintf(fp, "Querying On %"PRId64" records:\n", totalData);
H
hzcheng 已提交
6088 6089

  for (int j = 0; j < n; j++) {
6090
    char condition[COND_BUF_LEN] = "\0";
B
Bomin Zhang 已提交
6091
    char tempS[64] = "\0";
H
hzcheng 已提交
6092

6093
    int64_t m = 10 < num_of_tables ? 10 : num_of_tables;
H
hzcheng 已提交
6094

6095
    for (int64_t i = 1; i <= m; i++) {
H
hzcheng 已提交
6096
      if (i == 1) {
6097
        sprintf(tempS, "t1 = %"PRId64"", i);
H
hzcheng 已提交
6098
      } else {
6099
        sprintf(tempS, " or t1 = %"PRId64" ", i);
H
hzcheng 已提交
6100
      }
6101
      strncat(condition, tempS, COND_BUF_LEN - 1);
H
hzcheng 已提交
6102

L
liu0x54 已提交
6103
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
6104 6105 6106 6107

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

6108
      double t = taosGetTimestampMs();
H
hzcheng 已提交
6109

S
Shuaiqiang Chang 已提交
6110 6111 6112 6113
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
6114
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6115
        taos_free_result(pSql);
H
hzcheng 已提交
6116
        taos_close(taos);
6117
        fclose(fp);
6118
        return NULL;
H
hzcheng 已提交
6119 6120
      }
      int count = 0;
6121
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6122 6123
        count++;
      }
6124
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6125

6126
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
6127 6128
              num_of_tables * num_of_DPT / (t * 1000.0), t);
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0);
H
hzcheng 已提交
6129

S
Shuaiqiang Chang 已提交
6130
      taos_free_result(pSql);
H
hzcheng 已提交
6131 6132 6133 6134
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
6135
#endif
H
hzcheng 已提交
6136 6137 6138
  return NULL;
}

6139 6140 6141
static void prompt()
{
  if (!g_args.answer_yes) {
6142
    printf("         Press enter key to continue or Ctrl-C to stop\n\n");
6143 6144 6145 6146
    (void)getchar();
  }
}

6147
static int insertTestProcess() {
6148

6149 6150 6151
  setupForAnsiEscape();
  int ret = printfInsertMeta();
  resetAfterAnsiEscape();
6152

6153 6154 6155
  if (ret == -1)
    exit(EXIT_FAILURE);

6156
  debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
6157 6158
  g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
  if (NULL == g_fpOfInsertResult) {
6159
    errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
6160 6161
    return -1;
  }
6162

6163 6164
  if (g_fpOfInsertResult)
    printfInsertMetaToFile(g_fpOfInsertResult);
6165

6166
  prompt();
6167

6168 6169 6170
  init_rand_data();

  // create database and super tables
6171
  if(createDatabasesAndStables() != 0) {
6172 6173
    if (g_fpOfInsertResult)
      fclose(g_fpOfInsertResult);
6174 6175
    return -1;
  }
6176 6177

  // pretreatement
6178
  prepareSampleData();
6179

6180 6181 6182 6183
  double start;
  double end;

  // create child tables
6184
  start = taosGetTimestampMs();
6185
  createChildTables();
6186
  end = taosGetTimestampMs();
6187

6188
  if (g_totalChildTables > 0) {
6189
    fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6190
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6191 6192
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
6193
            "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6194
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6195
    }
6196
  }
6197

6198
  taosMsleep(1000);
6199
  // create sub threads for inserting data
6200
  //start = taosGetTimestampMs();
6201
  for (int i = 0; i < g_Dbs.dbCount; i++) {
6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
        for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {

          SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];

          if (superTblInfo && (superTblInfo->insertRows > 0)) {
            startMultiThreadInsertData(
              g_Dbs.threadCount,
              g_Dbs.db[i].dbName,
              g_Dbs.db[i].dbCfg.precision,
              superTblInfo);
          }
6215
        }
6216
      }
6217 6218
    } else {
        startMultiThreadInsertData(
6219 6220 6221
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
6222
          NULL);
H
Hui Li 已提交
6223
    }
6224
  }
6225
  //end = taosGetTimestampMs();
6226

6227
  //int64_t    totalInsertRows = 0;
6228
  //int64_t    totalAffectedRows = 0;
6229
  //for (int i = 0; i < g_Dbs.dbCount; i++) {
6230
  //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6231
  //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
6232 6233
  //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
  //}
6234
  //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
6235
  postFreeResource();
6236

6237 6238 6239
  return 0;
}

6240 6241
static void *specifiedTableQuery(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6242

6243
  if (pThreadInfo->taos == NULL) {
6244 6245 6246 6247 6248 6249 6250 6251
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6252
            pThreadInfo->threadID, taos_errstr(NULL));
6253 6254
      return NULL;
    } else {
6255
      pThreadInfo->taos = taos;
6256 6257 6258
    }
  }

6259 6260
  char sqlStr[MAX_DB_NAME_SIZE + 5];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6261 6262
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6263 6264 6265 6266
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
    return NULL;
  }
6267

6268 6269
  uint64_t st = 0;
  uint64_t et = 0;
6270

6271
  uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes;
6272

6273 6274 6275
  uint64_t totalQueried = 0;
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
6276

6277
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') {
6278
    sprintf(pThreadInfo->filePath, "%s-%d",
6279 6280 6281 6282
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }

6283
  while(queryTimes --) {
6284 6285 6286
    if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) <
            (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms
6287 6288
    }

6289 6290 6291
    st = taosGetTimestampMs();

    selectAndGetResult(pThreadInfo,
6292
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]);
6293

6294 6295 6296 6297
    et = taosGetTimestampMs();
    printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n",
              taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0);

6298 6299
    totalQueried ++;
    g_queryInfo.specifiedQueryInfo.totalQueried ++;
6300

6301 6302
    uint64_t  currentPrintTime = taosGetTimestampMs();
    uint64_t  endTs = taosGetTimestampMs();
6303
    if (currentPrintTime - lastPrintTime > 30*1000) {
6304
      debugPrint("%s() LN%d, endTs=%"PRIu64"ms, startTs=%"PRIu64"ms\n",
6305
          __func__, __LINE__, endTs, startTs);
6306
      printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.6f\n",
6307 6308
                    pThreadInfo->threadID,
                    totalQueried,
6309
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6310
      lastPrintTime = currentPrintTime;
6311
    }
H
Hui Li 已提交
6312
  }
6313 6314
  return NULL;
}
H
Hui Li 已提交
6315

6316
static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) {
6317 6318
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
6319 6320
  sprintf(subTblName, "%s.%s",
          g_queryInfo.dbName,
6321
          g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
6322 6323

  //printf("inSql: %s\n", inSql);
6324

6325 6326
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
6327
    return;
H
Hui Li 已提交
6328
  }
6329

6330
  tstrncpy(outSql, inSql, pos - inSql + 1);
6331
  //printf("1: %s\n", outSql);
6332
  strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1);
6333
  //printf("2: %s\n", outSql);
6334
  strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1);
6335
  //printf("3: %s\n", outSql);
H
Hui Li 已提交
6336 6337
}

6338
static void *superTableQuery(void *sarg) {
6339
  char sqlstr[MAX_QUERY_SQL_LENGTH];
6340
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6341

6342
  if (pThreadInfo->taos == NULL) {
6343 6344 6345 6346 6347 6348 6349 6350
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6351
            pThreadInfo->threadID, taos_errstr(NULL));
6352 6353
      return NULL;
    } else {
6354
      pThreadInfo->taos = taos;
6355 6356 6357
    }
  }

6358 6359
  uint64_t st = 0;
  uint64_t et = (int64_t)g_queryInfo.superQueryInfo.queryInterval;
6360

6361 6362 6363
  uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes;
  uint64_t totalQueried = 0;
  uint64_t  startTs = taosGetTimestampMs();
6364

6365
  uint64_t  lastPrintTime = taosGetTimestampMs();
6366
  while(queryTimes --) {
6367 6368 6369
    if (g_queryInfo.superQueryInfo.queryInterval
            && (et - st) < (int64_t)g_queryInfo.superQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.superQueryInfo.queryInterval - (et - st)); // ms
6370
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to);
6371
    }
H
Hui Li 已提交
6372

6373
    st = taosGetTimestampMs();
6374
    for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) {
6375
      for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
6376
        memset(sqlstr,0,sizeof(sqlstr));
6377
        replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i);
6378
        if (g_queryInfo.superQueryInfo.result[j][0] != '\0') {
6379
          sprintf(pThreadInfo->filePath, "%s-%d",
6380
                  g_queryInfo.superQueryInfo.result[j],
6381
                  pThreadInfo->threadID);
6382
        }
6383
        selectAndGetResult(pThreadInfo, sqlstr);
6384 6385 6386 6387 6388 6389 6390

        totalQueried++;
        g_queryInfo.superQueryInfo.totalQueried ++;

        int64_t  currentPrintTime = taosGetTimestampMs();
        int64_t  endTs = taosGetTimestampMs();
        if (currentPrintTime - lastPrintTime > 30*1000) {
6391
          printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.3f\n",
6392 6393
                    pThreadInfo->threadID,
                    totalQueried,
6394
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6395
          lastPrintTime = currentPrintTime;
6396
        }
6397
      }
H
Hui Li 已提交
6398
    }
6399
    et = taosGetTimestampMs();
6400
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%"PRIu64" - %"PRIu64"] once queries duration:%.4fs\n\n",
6401
            taosGetSelfPthreadId(),
6402 6403
            pThreadInfo->start_table_from,
            pThreadInfo->end_table_to,
6404
            (double)(et - st)/1000.0);
6405
  }
6406

6407 6408 6409
  return NULL;
}

6410
static int queryTestProcess() {
6411 6412 6413 6414

  setupForAnsiEscape();
  printfQueryMeta();
  resetAfterAnsiEscape();
6415 6416 6417 6418 6419 6420

  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
6421
          g_queryInfo.port);
6422
  if (taos == NULL) {
6423 6424
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6425 6426 6427
    exit(-1);
  }

6428
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6429 6430
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6431 6432 6433
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6434
  }
6435

6436
  prompt();
6437

6438 6439 6440
  if (g_args.debug_print || g_args.verbose_print) {
    printfQuerySystemInfo(taos);
  }
6441

6442 6443 6444 6445 6446 6447
  if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
    if (convertHostToServAddr(
        g_queryInfo.host, g_queryInfo.port, &g_queryInfo.serv_addr) != 0)
      exit(-1);
  }

6448 6449 6450
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
6451
  int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent;
6452
  uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount;
6453

6454
  uint64_t startTs = taosGetTimestampMs();
6455

6456 6457 6458 6459 6460 6461
  if ((nSqlCount > 0) && (nConcurrent > 0)) {

    pids  = malloc(nConcurrent * nSqlCount * sizeof(pthread_t));
    infos = malloc(nConcurrent * nSqlCount * sizeof(threadInfo));

    if ((NULL == pids) || (NULL == infos)) {
6462
      taos_close(taos);
6463
      ERROR_EXIT("memory allocation failed for create threads\n");
6464
    }
6465

6466 6467 6468
    for (uint64_t i = 0; i < nSqlCount; i++) {
        for (int j = 0; j < nConcurrent; j++) {
            uint64_t seq = i * nConcurrent + j;
6469 6470 6471
            threadInfo *pThreadInfo = infos + seq;
            pThreadInfo->threadID = seq;
            pThreadInfo->querySeq = i;
6472 6473 6474 6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485

            if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {

                char sqlStr[MAX_TB_NAME_SIZE*2];
                sprintf(sqlStr, "use %s", g_queryInfo.dbName);
                if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) {
                    taos_close(taos);
                    free(infos);
                    free(pids);
                    errorPrint( "use database %s failed!\n\n",
                        g_queryInfo.dbName);
                    return -1;
                }
            }
6486

6487
            pThreadInfo->taos = NULL;// TODO: workaround to use separate taos connection;
6488

6489
            pthread_create(pids + seq, NULL, specifiedTableQuery,
6490
                pThreadInfo);
6491
        }
6492
    }
6493
  } else {
6494
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
6495
  }
6496

6497 6498
  taos_close(taos);

6499 6500 6501
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
6502 6503 6504 6505
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
    pidsOfSub  = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t));
    infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo));
6506 6507

    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
6508 6509
      free(infos);
      free(pids);
6510

6511
      ERROR_EXIT("memory allocation failed for create threads\n");
6512
    }
6513

6514
    int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
6515
    int threads = g_queryInfo.superQueryInfo.threadCnt;
6516

6517
    int64_t a = ntables / threads;
6518 6519 6520 6521
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
6522

6523
    int64_t b = 0;
6524 6525 6526
    if (threads != 0) {
      b = ntables % threads;
    }
6527

6528
    uint64_t startFrom = 0;
6529
    for (int i = 0; i < threads; i++) {
6530 6531
      threadInfo *pThreadInfo = infosOfSub + i;
      pThreadInfo->threadID = i;
6532

6533 6534 6535 6536 6537 6538
      pThreadInfo->start_table_from = startFrom;
      pThreadInfo->ntables = i<b?a+1:a;
      pThreadInfo->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = pThreadInfo->end_table_to + 1;
      pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection;
      pthread_create(pidsOfSub + i, NULL, superTableQuery, pThreadInfo);
6539 6540
    }

6541
    g_queryInfo.superQueryInfo.threadCnt = threads;
6542
  } else {
6543
    g_queryInfo.superQueryInfo.threadCnt = 0;
6544
  }
6545

6546 6547 6548 6549 6550 6551
  if ((nSqlCount > 0) && (nConcurrent > 0)) {
    for (int i = 0; i < nConcurrent; i++) {
      for (int j = 0; j < nSqlCount; j++) {
        pthread_join(pids[i * nSqlCount + j], NULL);
      }
    }
6552
  }
S
Shuaiqiang Chang 已提交
6553

6554
  tmfree((char*)pids);
6555
  tmfree((char*)infos);
6556

6557
  for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
6558
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
6559
  }
H
Hui Li 已提交
6560

6561
  tmfree((char*)pidsOfSub);
6562
  tmfree((char*)infosOfSub);
6563

6564
//  taos_close(taos);// TODO: workaround to use separate taos connection;
6565
  uint64_t endTs = taosGetTimestampMs();
6566

6567
  uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried +
6568 6569
    g_queryInfo.superQueryInfo.totalQueried;

6570
  fprintf(stderr, "==== completed total queries: %"PRIu64", the QPS of all threads: %10.3f====\n",
6571
          totalQueried,
6572
          (double)(totalQueried/((endTs-startTs)/1000.0)));
6573 6574 6575
  return 0;
}

6576 6577 6578 6579 6580 6581 6582 6583 6584
static void stable_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
  if (res == NULL || taos_errno(res) != 0) {
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
    return;
  }

  if (param)
6585
    fetchResult(res, (threadInfo *)param);
6586 6587 6588 6589 6590
  // tao_unscribe() will free result.
}

static void specified_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
6591
  if (res == NULL || taos_errno(res) != 0) {
6592 6593
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
6594 6595
    return;
  }
6596

6597
  if (param)
6598
    fetchResult(res, (threadInfo *)param);
6599
  // tao_unscribe() will free result.
H
hzcheng 已提交
6600 6601
}

6602
static TAOS_SUB* subscribeImpl(
6603
        QUERY_CLASS class,
6604
        threadInfo *pThreadInfo,
6605
        char *sql, char* topic, bool restart, uint64_t interval)
6606
{
6607
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
6608

6609 6610
  if ((SPECIFIED_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) {
6611 6612
    tsub = taos_subscribe(
            pThreadInfo->taos,
6613
            restart,
6614
            topic, sql, specified_sub_callback, (void*)pThreadInfo,
6615
            g_queryInfo.specifiedQueryInfo.subscribeInterval);
6616 6617
  } else if ((STABLE_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) {
6618 6619 6620 6621 6622
    tsub = taos_subscribe(
            pThreadInfo->taos,
            restart,
            topic, sql, stable_sub_callback, (void*)pThreadInfo,
            g_queryInfo.superQueryInfo.subscribeInterval);
6623
  } else {
6624 6625
    tsub = taos_subscribe(
            pThreadInfo->taos,
6626
            restart,
6627
            topic, sql, NULL, NULL, interval);
6628
  }
6629

6630 6631 6632
  if (tsub == NULL) {
    printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
    return NULL;
6633
  }
6634

6635 6636
  return tsub;
}
H
hzcheng 已提交
6637

6638 6639
static void *superSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6640
  char subSqlstr[MAX_QUERY_SQL_LENGTH];
6641
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT] = {0};
6642
  uint64_t tsubSeq;
H
hzcheng 已提交
6643

6644 6645
  if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
      errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n",
6646 6647 6648 6649 6650
              pThreadInfo->ntables,
              MAX_QUERY_SQL_COUNT);
      exit(-1);
  }

6651
  if (pThreadInfo->taos == NULL) {
6652
    pThreadInfo->taos = taos_connect(g_queryInfo.host,
6653 6654 6655 6656
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
6657
    if (pThreadInfo->taos == NULL) {
6658
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6659
            pThreadInfo->threadID, taos_errstr(NULL));
6660 6661 6662 6663
      return NULL;
    }
  }

6664 6665
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6666 6667
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6668 6669
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
6670 6671
    return NULL;
  }
6672

6673 6674 6675
  char topic[32] = {0};
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6676 6677

      tsubSeq = i - pThreadInfo->start_table_from;
6678 6679 6680 6681 6682
      verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n",
              __func__, __LINE__,
              pThreadInfo->threadID,
              pThreadInfo->start_table_from,
              pThreadInfo->end_table_to, i);
6683 6684
      sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"",
              i, pThreadInfo->querySeq);
6685
      memset(subSqlstr, 0, sizeof(subSqlstr));
6686 6687 6688 6689
      replaceChildTblName(
              g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq],
              subSqlstr, i);
      if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6690
        sprintf(pThreadInfo->filePath, "%s-%d",
6691 6692 6693 6694 6695 6696
                g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
      }

      debugPrint("%s() LN%d, [%d] subSqlstr: %s\n",
              __func__, __LINE__, pThreadInfo->threadID, subSqlstr);
6697
      tsub[tsubSeq] = subscribeImpl(
6698
              STABLE_CLASS,
6699
              pThreadInfo, subSqlstr, topic,
6700 6701
              g_queryInfo.superQueryInfo.subscribeRestart,
              g_queryInfo.superQueryInfo.subscribeInterval);
6702
      if (NULL == tsub[tsubSeq]) {
6703
        taos_close(pThreadInfo->taos);
6704 6705
        return NULL;
      }
6706
  }
6707 6708

  // start loop to consume result
6709 6710 6711 6712
  int consumed[MAX_QUERY_SQL_COUNT];
  for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) {
    consumed[i] = 0;
  }
6713
  TAOS_RES* res = NULL;
6714

6715 6716
  uint64_t st = 0, et = 0;

6717 6718 6719 6720
  while ((g_queryInfo.superQueryInfo.endAfterConsume == -1)
          || (g_queryInfo.superQueryInfo.endAfterConsume <
              consumed[pThreadInfo->end_table_to - pThreadInfo->start_table_from])) {

6721 6722
    for (uint64_t i = pThreadInfo->start_table_from;
            i <= pThreadInfo->end_table_to; i++) {
6723 6724 6725 6726
      tsubSeq = i - pThreadInfo->start_table_from;
      if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
          continue;
      }
6727

6728 6729
      st = taosGetTimestampMs();
      performancePrint("st: %"PRIu64" et: %"PRIu64" st-et: %"PRIu64"\n", st, et, (st - et));
6730
      res = taos_consume(tsub[tsubSeq]);
6731 6732 6733
      et = taosGetTimestampMs();
      performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st));

6734 6735
      if (res) {
          if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6736
              sprintf(pThreadInfo->filePath, "%s-%d",
6737 6738
                      g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
6739
              fetchResult(res, pThreadInfo);
6740 6741
          }
          if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6742
              sprintf(pThreadInfo->filePath, "%s-%d",
6743 6744
                      g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
6745
              fetchResult(res, pThreadInfo);
6746 6747 6748
          }
          consumed[tsubSeq] ++;

6749
          if ((g_queryInfo.superQueryInfo.resubAfterConsume != -1)
6750
                  && (consumed[tsubSeq] >=
6751
                      g_queryInfo.superQueryInfo.resubAfterConsume)) {
6752 6753 6754 6755
              printf("keepProgress:%d, resub super table query: %"PRIu64"\n",
                      g_queryInfo.superQueryInfo.subscribeKeepProgress,
                      pThreadInfo->querySeq);
              taos_unsubscribe(tsub[tsubSeq],
6756
                    g_queryInfo.superQueryInfo.subscribeKeepProgress);
6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769
              consumed[tsubSeq]= 0;
              tsub[tsubSeq] = subscribeImpl(
                      STABLE_CLASS,
                      pThreadInfo, subSqlstr, topic,
                      g_queryInfo.superQueryInfo.subscribeRestart,
                      g_queryInfo.superQueryInfo.subscribeInterval
                      );
              if (NULL == tsub[tsubSeq]) {
                  taos_close(pThreadInfo->taos);
                  return NULL;
              }
          }
      }
H
hzcheng 已提交
6770 6771
    }
  }
6772
  taos_free_result(res);
6773

6774 6775
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6776 6777
    tsubSeq = i - pThreadInfo->start_table_from;
    taos_unsubscribe(tsub[tsubSeq], 0);
6778
  }
6779

6780
  taos_close(pThreadInfo->taos);
H
hzcheng 已提交
6781 6782 6783
  return NULL;
}

6784 6785
static void *specifiedSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6786
//  TAOS_SUB*  tsub = NULL;
6787

6788
  if (pThreadInfo->taos == NULL) {
6789
    pThreadInfo->taos = taos_connect(g_queryInfo.host,
6790 6791 6792 6793
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
6794
    if (pThreadInfo->taos == NULL) {
6795
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6796
            pThreadInfo->threadID, taos_errstr(NULL));
6797 6798 6799 6800
      return NULL;
    }
  }

6801 6802
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6803 6804
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6805 6806
    return NULL;
  }
6807

6808 6809 6810 6811
  sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
          "taosdemo-subscribe-%"PRIu64"-%d",
          pThreadInfo->querySeq,
          pThreadInfo->threadID);
6812
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') {
6813
      sprintf(pThreadInfo->filePath, "%s-%d",
6814
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
6815
                pThreadInfo->threadID);
6816
  }
6817 6818
  g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl(
          SPECIFIED_CLASS, pThreadInfo,
6819
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
6820
          g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
6821 6822
          g_queryInfo.specifiedQueryInfo.subscribeRestart,
          g_queryInfo.specifiedQueryInfo.subscribeInterval);
6823
  if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) {
6824 6825
      taos_close(pThreadInfo->taos);
      return NULL;
6826
  }
6827

6828
  // start loop to consume result
6829

6830
  g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0;
6831 6832 6833 6834
  while((g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq] == -1)
          || (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] <
              g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq])) {

6835
      if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) {
6836 6837
        continue;
      }
6838

6839 6840 6841
      g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] = taos_consume(
              g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]);
      if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) {
6842
          if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6843
              sprintf(pThreadInfo->filePath, "%s-%d",
6844 6845
                      g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
6846
              fetchResult(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], pThreadInfo);
6847 6848
          }

6849
          g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] ++;
6850
          if ((g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq] != -1)
6851
                && (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] >=
6852 6853 6854 6855
                    g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
              printf("keepProgress:%d, resub specified query: %"PRIu64"\n",
                    g_queryInfo.specifiedQueryInfo.subscribeKeepProgress,
                    pThreadInfo->querySeq);
6856 6857
              g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0;
              taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID],
6858
                      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
6859
              g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl(
6860 6861 6862
                      SPECIFIED_CLASS,
                      pThreadInfo,
                      g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
6863
                      g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
6864 6865
                      g_queryInfo.specifiedQueryInfo.subscribeRestart,
                      g_queryInfo.specifiedQueryInfo.subscribeInterval);
6866
              if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) {
6867 6868
                taos_close(pThreadInfo->taos);
                return NULL;
6869 6870
              }
          }
6871 6872
      }
  }
6873 6874
  taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]);
  taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->querySeq], 0);
6875
  taos_close(pThreadInfo->taos);
6876

H
hzcheng 已提交
6877 6878 6879
  return NULL;
}

6880
static int subscribeTestProcess() {
6881
  setupForAnsiEscape();
6882
  printfQueryMeta();
6883
  resetAfterAnsiEscape();
6884

6885
  prompt();
6886

6887 6888 6889 6890 6891
  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
6892
          g_queryInfo.port);
6893
  if (taos == NULL) {
6894 6895
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6896 6897 6898
    exit(-1);
  }

6899
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6900 6901
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6902 6903 6904
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6905 6906
  }

6907 6908
  taos_close(taos); // TODO: workaround to use separate taos connection;

6909 6910
  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
6911 6912 6913 6914 6915

  pthread_t  *pidsOfStable  = NULL;
  threadInfo *infosOfStable = NULL;

  //==== create threads for query for specified table
6916
  if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) {
6917
    debugPrint("%s() LN%d, sepcified query sqlCount %d.\n",
6918
              __func__, __LINE__,
6919 6920 6921
              g_queryInfo.specifiedQueryInfo.sqlCount);
  } else {
    if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) {
6922
        errorPrint("%s() LN%d, sepcified query sqlCount %d.\n",
6923 6924 6925 6926
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount);
        exit(-1);
    }
6927

6928
    pids  = malloc(
6929 6930 6931
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(pthread_t));
6932
    infos = malloc(
6933 6934 6935
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(threadInfo));
6936 6937 6938 6939
    if ((NULL == pids) || (NULL == infos)) {
        errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
        exit(-1);
    }
6940

6941 6942 6943
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
            uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
6944 6945 6946 6947 6948
            threadInfo *pThreadInfo = infos + seq;
            pThreadInfo->threadID = seq;
            pThreadInfo->querySeq = i;
            pThreadInfo->taos = NULL;  // TODO: workaround to use separate taos connection;
            pthread_create(pids + seq, NULL, specifiedSubscribe, pThreadInfo);
6949
        }
6950
    }
H
hzcheng 已提交
6951
  }
6952

6953
  //==== create threads for super table query
6954
  if (g_queryInfo.superQueryInfo.sqlCount <= 0) {
6955
    debugPrint("%s() LN%d, super table query sqlCount %d.\n",
6956
              __func__, __LINE__,
6957
              g_queryInfo.superQueryInfo.sqlCount);
6958 6959
  } else {
    if ((g_queryInfo.superQueryInfo.sqlCount > 0)
6960
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
6961 6962 6963
        pidsOfStable  = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
6964
            sizeof(pthread_t));
6965 6966 6967
        infosOfStable = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
6968
            sizeof(threadInfo));
6969 6970
        if ((NULL == pidsOfStable) || (NULL == infosOfStable)) {
            errorPrint("%s() LN%d, malloc failed for create threads\n",
6971
              __func__, __LINE__);
6972 6973 6974
            // taos_close(taos);
            exit(-1);
        }
6975

6976 6977
        int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
        int threads = g_queryInfo.superQueryInfo.threadCnt;
6978

6979 6980 6981 6982 6983
        int64_t a = ntables / threads;
        if (a < 1) {
            threads = ntables;
            a = 1;
        }
6984

6985 6986 6987 6988
        int64_t b = 0;
        if (threads != 0) {
            b = ntables % threads;
        }
6989

6990
        for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
6991
            uint64_t startFrom = 0;
6992 6993
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
6994 6995 6996 6997 6998 6999 7000 7001 7002
                threadInfo *pThreadInfo = infosOfStable + seq;
                pThreadInfo->threadID = seq;
                pThreadInfo->querySeq = i;

                pThreadInfo->start_table_from = startFrom;
                pThreadInfo->ntables = j<b?a+1:a;
                pThreadInfo->end_table_to = j<b?startFrom+a:startFrom+a-1;
                startFrom = pThreadInfo->end_table_to + 1;
                pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection;
7003
                pthread_create(pidsOfStable + seq,
7004
                        NULL, superSubscribe, pThreadInfo);
7005 7006
            }
        }
7007

7008
        g_queryInfo.superQueryInfo.threadCnt = threads;
7009

7010 7011 7012 7013 7014 7015
        for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                pthread_join(pidsOfStable[seq], NULL);
            }
        }
7016
    }
H
hzcheng 已提交
7017
  }
7018

7019 7020 7021 7022 7023
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
    for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
        uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
        pthread_join(pids[seq], NULL);
    }
7024
  }
H
hzcheng 已提交
7025

7026
  tmfree((char*)pids);
7027
  tmfree((char*)infos);
H
hzcheng 已提交
7028

7029 7030
  tmfree((char*)pidsOfStable);
  tmfree((char*)infosOfStable);
7031
//   taos_close(taos);
7032
  return 0;
H
hzcheng 已提交
7033 7034
}

7035
static void initOfInsertMeta() {
7036
  memset(&g_Dbs, 0, sizeof(SDbs));
7037

7038
  // set default values
7039
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7040
  g_Dbs.port = 6030;
7041 7042
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
7043
  g_Dbs.threadCount = 2;
7044 7045

  g_Dbs.use_metric = g_args.use_metric;
L
Liu Tao 已提交
7046 7047
}

7048
static void initOfQueryMeta() {
7049
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
7050

7051
  // set default values
7052
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7053
  g_queryInfo.port = 6030;
7054 7055
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
L
Liu Tao 已提交
7056 7057
}

7058
static void setParaFromArg(){
7059
  if (g_args.host) {
7060
    tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE);
7061
  } else {
7062
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7063
  }
L
Liu Tao 已提交
7064

7065
  if (g_args.user) {
7066
    tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE);
7067
  }
7068 7069

  if (g_args.password) {
7070
    tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE);
7071 7072
  }

7073 7074
  if (g_args.port) {
    g_Dbs.port = g_args.port;
7075
  }
L
Liu Tao 已提交
7076

7077 7078 7079
  g_Dbs.threadCount = g_args.num_of_threads;
  g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;

7080 7081
  g_Dbs.dbCount = 1;
  g_Dbs.db[0].drop = 1;
7082

7083
  tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE);
7084
  g_Dbs.db[0].dbCfg.replica = g_args.replica;
7085
  tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
7086

7087
  tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
7088 7089 7090 7091 7092

  g_Dbs.use_metric = g_args.use_metric;
  g_Dbs.insert_only = g_args.insert_only;

  g_Dbs.do_aggreFunc = true;
L
Liu Tao 已提交
7093

7094 7095
  char dataString[STRING_LEN];
  char **data_type = g_args.datatype;
7096

7097
  memset(dataString, 0, STRING_LEN);
L
Liu Tao 已提交
7098

7099 7100
  if (strcasecmp(data_type[0], "BINARY") == 0
          || strcasecmp(data_type[0], "BOOL") == 0
7101
          || strcasecmp(data_type[0], "NCHAR") == 0 ) {
7102
    g_Dbs.do_aggreFunc = false;
L
Liu Tao 已提交
7103
  }
H
hzcheng 已提交
7104

7105 7106 7107 7108 7109
  if (g_args.use_metric) {
    g_Dbs.db[0].superTblCount = 1;
    tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE);
    g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
    g_Dbs.threadCount = g_args.num_of_threads;
7110
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
7111
    g_Dbs.asyncMode = g_args.async_mode;
7112

7113 7114 7115 7116
    g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
    g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
    g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
7117
    tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix,
7118 7119 7120
            g_args.tb_prefix, MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE);
7121
    tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp,
7122
            "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
7123
    g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
7124

7125
    g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
7126
    g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len;
7127

7128 7129 7130 7131 7132
    g_Dbs.db[0].superTbls[0].columnCount = 0;
    for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
      if (data_type[i] == NULL) {
        break;
      }
7133

7134
      tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
7135
              data_type[i], MAX_TB_NAME_SIZE);
7136
      g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;
7137 7138
      g_Dbs.db[0].superTbls[0].columnCount++;
    }
7139

7140 7141 7142
    if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
      g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
    } else {
7143 7144 7145 7146
      for (int i = g_Dbs.db[0].superTbls[0].columnCount;
              i < g_args.num_of_CPR; i++) {
        tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
                "INT", MAX_TB_NAME_SIZE);
7147
        g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;
7148 7149 7150
        g_Dbs.db[0].superTbls[0].columnCount++;
      }
    }
7151

7152 7153
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType,
            "INT", MAX_TB_NAME_SIZE);
7154
    g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;
7155

7156 7157
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType,
            "BINARY", MAX_TB_NAME_SIZE);
7158 7159
    g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;
    g_Dbs.db[0].superTbls[0].tagCount = 2;
7160
  } else {
7161 7162
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
    g_Dbs.db[0].superTbls[0].tagCount = 0;
H
hzcheng 已提交
7163
  }
7164 7165 7166 7167 7168 7169
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
7170

7171 7172 7173
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
7174 7175 7176
    exit(-1);
  }

7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

7201
static void querySqlFile(TAOS* taos, char* sqlFile)
7202 7203 7204 7205 7206 7207
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
7208

7209 7210 7211 7212 7213 7214
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

7215
  double t = taosGetTimestampMs();
7216

7217
  while((read_len = tgetline(&line, &line_len, fp)) != -1) {
7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
7230
    }
7231 7232

    memcpy(cmd + cmd_len, line, read_len);
7233 7234 7235
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) {
        errorPrint("%s() LN%d, queryDbExec %s failed!\n",
               __func__, __LINE__, cmd);
7236 7237 7238 7239 7240
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
7241 7242
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
7243 7244
  }

7245
  t = taosGetTimestampMs() - t;
7246
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
7247

7248 7249 7250 7251
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
7252 7253
}

7254
static void testMetaFile() {
7255
    if (INSERT_TEST == g_args.test_mode) {
7256 7257
      if (g_Dbs.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
7258

7259
      insertTestProcess();
7260

7261
    } else if (QUERY_TEST == g_args.test_mode) {
7262 7263
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7264

7265
      queryTestProcess();
7266

7267
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
7268 7269
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7270

7271
      subscribeTestProcess();
7272

7273 7274 7275
    }  else {
      ;
    }
7276
}
7277

7278
static void queryResult() {
7279
  // query data
7280

7281
  pthread_t read_id;
7282 7283 7284 7285
  threadInfo *pThreadInfo = malloc(sizeof(threadInfo));
  assert(pThreadInfo);
  pThreadInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
  pThreadInfo->start_table_from = 0;
7286

7287
  //pThreadInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
7288
  if (g_args.use_metric) {
7289 7290 7291 7292
    pThreadInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
    pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
    pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
    tstrncpy(pThreadInfo->tb_prefix,
7293 7294
          g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE);
  } else {
7295 7296 7297
    pThreadInfo->ntables = g_args.num_of_tables;
    pThreadInfo->end_table_to = g_args.num_of_tables -1;
    tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE);
7298 7299
  }

7300
  pThreadInfo->taos = taos_connect(
7301 7302 7303 7304 7305
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
          g_Dbs.port);
7306
  if (pThreadInfo->taos == NULL) {
7307 7308
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
7309
    free(pThreadInfo);
7310 7311 7312
    exit(-1);
  }

7313
  tstrncpy(pThreadInfo->filePath, g_Dbs.resultFile, MAX_FILE_NAME_LEN);
7314 7315

  if (!g_Dbs.use_metric) {
7316
    pthread_create(&read_id, NULL, readTable, pThreadInfo);
7317
  } else {
7318
    pthread_create(&read_id, NULL, readMetric, pThreadInfo);
7319 7320
  }
  pthread_join(read_id, NULL);
7321 7322
  taos_close(pThreadInfo->taos);
  free(pThreadInfo);
7323 7324
}

7325 7326
static void testCmdLine() {

7327 7328 7329 7330 7331 7332 7333 7334 7335 7336
  if (strlen(configDir)) {
    wordexp_t full_path;
    if (wordexp(configDir, &full_path, 0) != 0) {
      errorPrint( "Invalid path %s\n", configDir);
      return;
    }
    taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
    wordfree(&full_path);
  }

7337 7338
  g_args.test_mode = INSERT_TEST;
  insertTestProcess();
7339

7340
  if (false == g_Dbs.insert_only)
7341
    queryResult();
7342 7343
}

7344 7345 7346
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

7347
  debugPrint("meta file: %s\n", g_args.metaFile);
7348 7349 7350

  if (g_args.metaFile) {
    initOfInsertMeta();
7351
    initOfQueryMeta();
7352 7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
7365 7366 7367 7368
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
7369
          g_Dbs.port);
7370
      querySqlFile(qtaos, g_args.sqlFile);
7371 7372 7373 7374 7375
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
7376 7377 7378

    if (g_dupstr)
        free(g_dupstr);
H
hzcheng 已提交
7379
  }
7380 7381

  return 0;
H
hzcheng 已提交
7382
}
7383