taosdemo.c 269.3 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

21
#include <stdint.h>
22
#include <taos.h>
S
slguan 已提交
23
#define _GNU_SOURCE
24 25
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
42
#else
S
TD-1057  
Shengliang Guan 已提交
43 44
  #include <regex.h>
  #include <stdio.h>
45
#endif
H
hzcheng 已提交
46

47
#include <assert.h>
48
#include <stdlib.h>
49 50
#include "cJSON.h"

51
#include "os.h"
H
hzcheng 已提交
52
#include "taos.h"
53
#include "taoserror.h"
H
Hui Li 已提交
54
#include "tutil.h"
S
slguan 已提交
55

56 57
#define STMT_IFACE_ENABLED  1

58 59 60
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
61 62
extern char configDir[];

63 64 65 66
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

67 68
#define STR_INSERT_INTO     "INSERT INTO "

69 70 71 72 73 74
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};
75

76 77
#define MAX_RECORDS_PER_REQ     32766

78 79
#define HEAD_BUFF_LEN    1024*24  // 16*1024 + (192+32)*2 + insert into ..

80 81
#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
82
#define COND_BUF_LEN       (BUFFER_SIZE - 30)
83 84 85
#define MAX_USERNAME_SIZE  64
#define MAX_PASSWORD_SIZE  64
#define MAX_HOSTNAME_SIZE  64
86
#define MAX_TB_NAME_SIZE   64
87
#define MAX_DATA_SIZE      (16*1024)+20     // max record len: 16*1024, timestamp string and ,('') need extra space
88 89 90 91
#define MAX_NUM_DATATYPE   10
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
92
#define MAX_FILE_NAME_LEN  256              // max file name length on linux is 255.
93 94 95 96 97

#define   MAX_SAMPLES_ONCE_FROM_FILE   10000
#define   MAX_NUM_DATATYPE 10

#define   MAX_DB_COUNT           8
98
#define   MAX_SUPER_TABLE_COUNT  200
99 100 101
#define   MAX_COLUMN_COUNT       1024
#define   MAX_TAG_COUNT          128

102
#define   MAX_QUERY_SQL_COUNT    100
103
#define   MAX_QUERY_SQL_LENGTH   1024
104 105

#define   MAX_DATABASE_COUNT     256
106
#define INPUT_BUF_LEN   256
107

108
#define DEFAULT_TIMESTAMP_STEP  1
109

110

111 112 113 114 115
typedef enum CREATE_SUB_TALBE_MOD_EN {
  PRE_CREATE_SUBTBL,
  AUTO_CREATE_SUBTBL,
  NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
116

117 118
typedef enum TALBE_EXISTS_EN {
  TBL_NO_EXISTS,
119
  TBL_ALREADY_EXISTS,
120 121 122
  TBL_EXISTS_BUTT
} TALBE_EXISTS_EN;

123
enum enumSYNC_MODE {
124 125
  SYNC_MODE,
  ASYNC_MODE,
126 127
  MODE_BUT
};
128

S
Shuduo Sang 已提交
129
enum enum_TAOS_INTERFACE {
130 131 132
    TAOSC_IFACE,
    REST_IFACE,
    STMT_IFACE,
133 134 135
    INTERFACE_BUT
};

136 137 138 139 140 141
typedef enum enumQUERY_CLASS {
    SPECIFIED_CLASS,
    STABLE_CLASS,
    CLASS_BUT
} QUERY_CLASS;

S
Shuduo Sang 已提交
142
typedef enum enum_PROGRESSIVE_OR_INTERLACE {
143 144 145
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
146
} PROG_OR_INTERLACE_MODE;
147

148
typedef enum enumQUERY_TYPE {
149
  NO_INSERT_TYPE,
150
  INSERT_TYPE,
151
  QUERY_TYPE_BUT
152
} QUERY_TYPE;
153 154 155 156 157 158 159

enum _show_db_index {
  TSDB_SHOW_DB_NAME_INDEX,
  TSDB_SHOW_DB_CREATED_TIME_INDEX,
  TSDB_SHOW_DB_NTABLES_INDEX,
  TSDB_SHOW_DB_VGROUPS_INDEX,
  TSDB_SHOW_DB_REPLICA_INDEX,
160
  TSDB_SHOW_DB_QUORUM_INDEX,
161 162 163 164 165 166 167 168 169 170
  TSDB_SHOW_DB_DAYS_INDEX,
  TSDB_SHOW_DB_KEEP_INDEX,
  TSDB_SHOW_DB_CACHE_INDEX,
  TSDB_SHOW_DB_BLOCKS_INDEX,
  TSDB_SHOW_DB_MINROWS_INDEX,
  TSDB_SHOW_DB_MAXROWS_INDEX,
  TSDB_SHOW_DB_WALLEVEL_INDEX,
  TSDB_SHOW_DB_FSYNC_INDEX,
  TSDB_SHOW_DB_COMP_INDEX,
  TSDB_SHOW_DB_CACHELAST_INDEX,
171
  TSDB_SHOW_DB_PRECISION_INDEX,
172 173 174 175 176 177 178 179 180 181
  TSDB_SHOW_DB_UPDATE_INDEX,
  TSDB_SHOW_DB_STATUS_INDEX,
  TSDB_MAX_SHOW_DB
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
  TSDB_SHOW_STABLES_NAME_INDEX,
  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
  TSDB_SHOW_STABLES_COLUMNS_INDEX,
182 183
  TSDB_SHOW_STABLES_METRIC_INDEX,
  TSDB_SHOW_STABLES_UID_INDEX,
184
  TSDB_SHOW_STABLES_TID_INDEX,
185
  TSDB_SHOW_STABLES_VGID_INDEX,
186
  TSDB_MAX_SHOW_STABLES
187 188
};

189 190 191 192 193 194 195 196
enum _describe_table_index {
  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
  TSDB_MAX_DESCRIBE_METRIC
};

H
hzcheng 已提交
197
/* Used by main to communicate with parse_opt. */
198 199
static char *g_dupstr = NULL;

200 201
typedef struct SArguments_S {
  char *   metaFile;
202
  uint32_t test_mode;
203 204
  char *   host;
  uint16_t port;
S
Shuduo Sang 已提交
205
  uint16_t iface;
206 207 208 209 210 211 212
  char *   user;
  char *   password;
  char *   database;
  int      replica;
  char *   tb_prefix;
  char *   sqlFile;
  bool     use_metric;
213
  bool     drop_database;
214
  bool     insert_only;
215
  bool     answer_yes;
216
  bool     debug_print;
217
  bool     verbose_print;
218
  bool     performance_print;
219
  char *   output_file;
220
  bool     async_mode;
221
  char *   datatype[MAX_NUM_DATATYPE + 1];
222 223 224 225
  uint32_t len_of_binary;
  uint32_t num_of_CPR;
  uint32_t num_of_threads;
  uint64_t insert_interval;
226
  int64_t  query_times;
227 228
  uint32_t interlace_rows;
  uint32_t num_of_RPR;                  // num_of_records_per_req
229
  uint64_t max_sql_len;
230 231
  int64_t  num_of_tables;
  int64_t  num_of_DPT;
232
  int      abort;
233
  uint32_t disorderRatio;               // 0: no disorder, >0: x%
234
  int      disorderRange;               // ms or us by database precision
235
  uint32_t method_of_delete;
236
  char **  arg_list;
237 238
  uint64_t totalInsertRows;
  uint64_t totalAffectedRows;
239 240 241
} SArguments;

typedef struct SColumn_S {
242 243
  char      field[TSDB_COL_NAME_LEN];
  char      dataType[16];
244
  uint32_t  dataLen;
245
  char      note[128];
246 247 248
} StrColumn;

typedef struct SSuperTable_S {
249 250 251
  char         sTblName[TSDB_TABLE_NAME_LEN];
  char         dataSource[MAX_TB_NAME_SIZE];  // rand_gen or sample
  char         childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq
252 253
  char         insertMode[MAX_TB_NAME_SIZE];    // taosc, rest
  uint16_t     childTblExists;
254
  int64_t      childTblCount;
255 256
  uint64_t     batchCreateTableNum;     // 0: no batch,  > 0: batch table number in one sql
  uint8_t      autoCreateTable;         // 0: create sub table, 1: auto create sub table
257
  uint16_t     iface;                   // 0: taosc, 1: rest, 2: stmt
258
  int64_t      childTblLimit;
259
  uint64_t     childTblOffset;
260

261
//  int          multiThreadWriteOneTbl;  // 0: no, 1: yes
262
  uint32_t     interlaceRows;           //
263 264
  int          disorderRatio;           // 0: no disorder, >0: x%
  int          disorderRange;           // ms or us by database precision
265
  uint64_t     maxSqlLen;               //
266

267
  uint64_t     insertInterval;          // insert interval, will override global insert interval
268
  int64_t      insertRows;
269
  int64_t      timeStampStep;
270
  char         startTimestamp[MAX_TB_NAME_SIZE];
271
  char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
272 273
  char         sampleFile[MAX_FILE_NAME_LEN];
  char         tagsFile[MAX_FILE_NAME_LEN];
274

275
  uint32_t     columnCount;
276
  StrColumn    columns[MAX_COLUMN_COUNT];
277
  uint32_t     tagCount;
278 279 280
  StrColumn    tags[MAX_TAG_COUNT];

  char*        childTblName;
281
  char*        colsOfCreateChildTable;
282 283
  uint64_t     lenOfOneRow;
  uint64_t     lenOfTagOfOneRow;
284 285 286 287 288

  char*        sampleDataBuf;
  //int          sampleRowCount;
  //int          sampleUsePos;

289
  uint32_t     tagSource;    // 0: rand, 1: tag sample
290
  char*        tagDataBuf;
291 292
  uint32_t     tagSampleCount;
  uint32_t     tagUsePos;
293

294
  // statistics
295 296
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
297 298 299
} SSuperTable;

typedef struct {
300
  char     name[TSDB_DB_NAME_LEN];
301
  char     create_time[32];
S
Shuduo Sang 已提交
302
  int64_t  ntables;
303
  int32_t  vgroups;
304 305
  int16_t  replica;
  int16_t  quorum;
306
  int16_t  days;
307 308 309 310 311 312 313 314 315 316 317 318 319 320
  char     keeplist[32];
  int32_t  cache; //MB
  int32_t  blocks;
  int32_t  minrows;
  int32_t  maxrows;
  int8_t   wallevel;
  int32_t  fsync;
  int8_t   comp;
  int8_t   cachelast;
  char     precision[8];   // time resolution
  int8_t   update;
  char     status[16];
} SDbInfo;

321
typedef struct SDbCfg_S {
322
//  int       maxtablesPerVnode;
323 324
  uint32_t  minRows;        // 0 means default
  uint32_t  maxRows;        // 0 means default
325 326
  int       comp;
  int       walLevel;
327
  int       cacheLast;
328
  int       fsync;
329 330 331 332 333 334 335
  int       replica;
  int       update;
  int       keep;
  int       days;
  int       cache;
  int       blocks;
  int       quorum;
336
  char      precision[8];
337 338 339
} SDbCfg;

typedef struct SDataBase_S {
340
  char         dbName[TSDB_DB_NAME_LEN];
341
  bool         drop;  // 0: use exists, 1: if exists, drop then new create
342
  SDbCfg       dbCfg;
343
  uint64_t     superTblCount;
344 345 346 347
  SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
} SDataBase;

typedef struct SDbs_S {
348
  char         cfgDir[MAX_FILE_NAME_LEN];
349
  char         host[MAX_HOSTNAME_SIZE];
350 351
  struct sockaddr_in serv_addr;

352
  uint16_t     port;
353 354
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
355
  char         resultFile[MAX_FILE_NAME_LEN];
356 357 358
  bool         use_metric;
  bool         insert_only;
  bool         do_aggreFunc;
359
  bool         asyncMode;
360

361 362 363
  uint32_t     threadCount;
  uint32_t     threadCountByCreateTbl;
  uint32_t     dbCount;
364 365 366
  SDataBase    db[MAX_DB_COUNT];

  // statistics
367 368
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
369

370 371
} SDbs;

372
typedef struct SpecifiedQueryInfo_S {
373
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
374
  uint32_t     concurrent;
375
  int          sqlCount;
376
  uint32_t     asyncMode; // 0: sync, 1: async
377 378
  uint64_t     subscribeInterval; // ms
  uint64_t     queryTimes;
379
  bool         subscribeRestart;
380
  int          subscribeKeepProgress;
381
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
382
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN];
383
  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
384
  int          endAfterConsume[MAX_QUERY_SQL_COUNT];
385
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
386 387 388
  char         topic[MAX_QUERY_SQL_COUNT][32];
  int          consumed[MAX_QUERY_SQL_COUNT];
  TAOS_RES*    res[MAX_QUERY_SQL_COUNT];
389
  uint64_t     totalQueried;
390
} SpecifiedQueryInfo;
391

392
typedef struct SuperQueryInfo_S {
393
  char         sTblName[TSDB_TABLE_NAME_LEN];
394 395
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint32_t     threadCnt;
396
  uint32_t     asyncMode; // 0: sync, 1: async
397
  uint64_t     subscribeInterval; // ms
398
  bool         subscribeRestart;
399
  int          subscribeKeepProgress;
400
  uint64_t     queryTimes;
401
  int64_t      childTblCount;
402
  char         childTblPrefix[TSDB_TABLE_NAME_LEN - 20];    // 20 characters reserved for seq
403
  int          sqlCount;
404
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
405
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN];
406
  int          resubAfterConsume;
407
  int          endAfterConsume;
408
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
409

410
  char*        childTblName;
411
  uint64_t     totalQueried;
412
} SuperQueryInfo;
413 414

typedef struct SQueryMetaInfo_S {
415
  char         cfgDir[MAX_FILE_NAME_LEN];
416
  char         host[MAX_HOSTNAME_SIZE];
417
  uint16_t     port;
418
  struct       sockaddr_in serv_addr;
419 420
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
421
  char         dbName[TSDB_DB_NAME_LEN];
422
  char         queryMode[MAX_TB_NAME_SIZE];  // taosc, rest
423

424
  SpecifiedQueryInfo  specifiedQueryInfo;
425
  SuperQueryInfo      superQueryInfo;
426
  uint64_t     totalQueried;
427 428 429
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
430
  TAOS *    taos;
431
  TAOS_STMT *stmt;
432
  int       threadID;
433
  char      db_name[TSDB_DB_NAME_LEN];
434
  uint32_t  time_precision;
435 436
  char      filePath[4096];
  FILE      *fp;
437
  char      tb_prefix[TSDB_TABLE_NAME_LEN];
438 439
  uint64_t  start_table_from;
  uint64_t  end_table_to;
S
Shuduo Sang 已提交
440
  int64_t   ntables;
441 442 443 444
  uint64_t  data_of_rate;
  int64_t   start_time;
  char*     cols;
  bool      use_metric;
445
  SSuperTable* superTblInfo;
S
Shuduo Sang 已提交
446
  char      *buffer;    // sql cmd buffer
447 448

  // for async insert
449 450
  tsem_t    lock_sem;
  int64_t   counter;
451 452
  uint64_t  st;
  uint64_t  et;
453
  uint64_t  lastTs;
454

455
  // sample data
456
  int64_t   samplePos;
457
  // statistics
458 459
  uint64_t  totalInsertRows;
  uint64_t  totalAffectedRows;
460 461

  // insert delay statistics
462 463 464 465 466
  uint64_t  cntDelay;
  uint64_t  totalDelay;
  uint64_t  avgDelay;
  uint64_t  maxDelay;
  uint64_t  minDelay;
467

468
  // seq of query or subscribe
469
  uint64_t  querySeq;   // sequence number of sql command
470
  TAOS_SUB*  tsub;
471

472 473
} threadInfo;

474
#ifdef WINDOWS
475 476
#define _CRT_RAND_S

477
#include <windows.h>
478 479 480 481 482
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
483 484
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
485 486
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
487 488 489 490

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

491
static void setupForAnsiEscape(void) {
492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
510
  }
511 512
}

513
static void resetAfterAnsiEscape(void) {
514
  // Reset colors
515
  printf("\x1b[0m");
516 517 518 519 520 521

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
522

523
static int taosRandom()
524 525 526 527 528 529
{
    int number;
    rand_s(&number);

    return number;
}
530
#else   // Not windows
531
static void setupForAnsiEscape(void) {}
532

533
static void resetAfterAnsiEscape(void) {
534 535 536
  // Reset colors
  printf("\x1b[0m");
}
537

538 539
#include <time.h>

540
static int taosRandom()
541
{
542
  return rand();
543 544
}

545
#endif // ifdef Windows
546

547
static void prompt();
548
static int createDatabasesAndStables();
549
static void createChildTables();
550
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet);
551
static int postProceSql(char *host, struct sockaddr_in *pServAddr,
552
        uint16_t port, char* sqlstr, threadInfo *pThreadInfo);
553 554
static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq,
        int disorderRatio, int disorderRange);
555 556 557 558 559 560 561

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
562
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)",
563 564
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

565
SArguments g_args = {
566 567 568 569
                     NULL,            // metaFile
                     0,               // test_mode
                     "127.0.0.1",     // host
                     6030,            // port
S
Shuduo Sang 已提交
570
                     TAOSC_IFACE,     // iface
571 572 573
                     "root",          // user
                     #ifdef _TD_POWER_
                     "powerdb",      // password
574 575
                     #elif (_TD_TQ_ == true)
                     "tqueue",      // password
576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604
                     #else
                     "taosdata",      // password
                     #endif
                     "test",          // database
                     1,               // replica
                     "t",             // tb_prefix
                     NULL,            // sqlFile
                     true,            // use_metric
                     true,            // drop_database
                     true,            // insert_only
                     false,           // debug_print
                     false,           // verbose_print
                     false,           // performance statistic print
                     false,           // answer_yes;
                     "./output.txt",  // output_file
                     0,               // mode : sync or async
                     {
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     },
                     16,              // len_of_binary
                     4,               // num_of_CPR
                     10,              // num_of_connections/thread
                     0,               // insert_interval
                     1,               // query_times
                     0,               // interlace_rows;
                     30000,           // num_of_RPR
605
                     (1024*1024),     // max_sql_len
606 607 608 609 610 611 612
                     10000,           // num_of_tables
                     10000,           // num_of_DPT
                     0,               // abort
                     0,               // disorderRatio
                     1000,            // disorderRange
                     1,               // method_of_delete
                     NULL             // arg_list
613 614 615 616 617
};



static SDbs            g_Dbs;
618
static int64_t         g_totalChildTables = 0;
619 620 621
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

622 623 624 625
#if _MSC_VER <= 1900
#define __func__ __FUNCTION__
#endif

626
#define debugPrint(fmt, ...) \
627
    do { if (g_args.debug_print || g_args.verbose_print) \
628
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
629

630
#define verbosePrint(fmt, ...) \
631 632
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
633

634 635 636 637
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)

638 639 640
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)

641 642 643
// for strncpy buffer overflow
#define min(a, b) (((a) < (b)) ? (a) : (b))

644

645
///////////////////////////////////////////////////
H
hzcheng 已提交
646

647 648
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

649 650 651 652 653 654 655 656
#ifndef TAOSDEMO_COMMIT_SHA1
#define TAOSDEMO_COMMIT_SHA1 "unknown"
#endif

#ifndef TD_VERNUMBER
#define TD_VERNUMBER    "unknown"
#endif

657
#ifndef TAOSDEMO_STATUS
658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674
#define TAOSDEMO_STATUS "unknown"
#endif

static void printVersion() {
    char tdengine_ver[] = TD_VERNUMBER;
    char taosdemo_ver[] = TAOSDEMO_COMMIT_SHA1;
    char taosdemo_status[] = TAOSDEMO_STATUS;

    if (strlen(taosdemo_status) == 0) {
        printf("taosdemo verison %s-%s\n",
                tdengine_ver, taosdemo_ver);
    } else {
        printf("taosdemo verison %s-%s, status:%s\n",
                tdengine_ver, taosdemo_ver, taosdemo_status);
    }
}

675
static void printHelp() {
676
  char indent[10] = "        ";
677
  printf("%s%s%s%s\n", indent, "-f", indent,
678
          "The meta file to the execution procedure. Default is './meta.json'.");
679
  printf("%s%s%s%s\n", indent, "-u", indent,
680 681
          "The TDengine user name to use when connecting to the server. Default is 'root'.");
#ifdef _TD_POWER_
682
  printf("%s%s%s%s\n", indent, "-P", indent,
683
          "The password to use when connecting to the server. Default is 'powerdb'.");
684
  printf("%s%s%s%s\n", indent, "-c", indent,
685
          "Configuration directory. Default is '/etc/power/'.");
686 687 688 689 690
#elif (_TD_TQ_ == true)
	printf("%s%s%s%s\n", indent, "-P", indent,
			"The password to use when connecting to the server. Default is 'tqueue'.");
	printf("%s%s%s%s\n", indent, "-c", indent,
			"Configuration directory. Default is '/etc/tq/'.");
691
#else
692
  printf("%s%s%s%s\n", indent, "-P", indent,
693
          "The password to use when connecting to the server. Default is 'taosdata'.");
694
  printf("%s%s%s%s\n", indent, "-c", indent,
695
          "Configuration directory. Default is '/etc/taos/'.");
696
#endif
697
  printf("%s%s%s%s\n", indent, "-h", indent,
698
          "The host to connect to TDengine. Default is localhost.");
699
  printf("%s%s%s%s\n", indent, "-p", indent,
700
          "The TCP/IP port number to use for the connection. Default is 0.");
701
  printf("%s%s%s%s\n", indent, "-I", indent,
702
#if STMT_IFACE_ENABLED == 1
703
          "The interface (taosc, rest, and stmt) taosdemo uses. Default is 'taosc'.");
704 705 706
#else
          "The interface (taosc, rest) taosdemo uses. Default is 'taosc'.");
#endif
707
  printf("%s%s%s%s\n", indent, "-d", indent,
708
          "Destination database. Default is 'test'.");
709
  printf("%s%s%s%s\n", indent, "-a", indent,
710
          "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
711
  printf("%s%s%s%s\n", indent, "-m", indent,
712 713
          "Table prefix name. Default is 't'.");
  printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
714
  printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
715
  printf("%s%s%s%s\n", indent, "-o", indent,
716
          "Direct output to the named file. Default is './output.txt'.");
717
  printf("%s%s%s%s\n", indent, "-q", indent,
718
          "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC.");
719
  printf("%s%s%s%s\n", indent, "-b", indent,
sangshuduo's avatar
sangshuduo 已提交
720
          "The data_type of columns, default: INT,INT,INT,INT.");
721
  printf("%s%s%s%s\n", indent, "-w", indent,
722
          "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
723 724 725
  printf("%s%s%s%s%d\n", indent, "-l", indent,
          "The number of columns per record. Default is 4. Max values is ",
       MAX_NUM_DATATYPE);
726
  printf("%s%s%s%s\n", indent, "-T", indent,
727
          "The number of threads. Default is 10.");
728
  printf("%s%s%s%s\n", indent, "-i", indent,
729
          "The sleep time (ms) between insertion. Default is 0.");
730
  printf("%s%s%s%s\n", indent, "-r", indent,
731
          "The number of records per request. Default is 30000.");
732
  printf("%s%s%s%s\n", indent, "-t", indent,
733
          "The number of tables. Default is 10000.");
734
  printf("%s%s%s%s\n", indent, "-n", indent,
735 736 737
          "The number of records per table. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
  printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
738
  printf("%s%s%s%s\n", indent, "-O", indent,
739
          "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order.");
740
  printf("%s%s%s%s\n", indent, "-R", indent,
741
          "Out of order data's range, ms, default is 1000.");
742
  printf("%s%s%s%s\n", indent, "-g", indent,
743
          "Print debug info.");
744
  printf("%s%s%s\n", indent, "-V, --version\t",
745
          "Print version info.");
746 747
  printf("%s%s%s%s\n", indent, "--help\t", indent,
          "Print command line arguments list info.");
748
/*    printf("%s%s%s%s\n", indent, "-D", indent,
749 750 751 752
          "if elete database if exists. 0: no, 1: yes, default is 1");
          */
}

753 754 755 756 757 758 759 760 761 762 763 764 765 766 767
static bool isStringNumber(char *input)
{
  int len = strlen(input);
  if (0 == len) {
    return false;
  }

  for (int i = 0; i < len; i++) {
    if (!isdigit(input[i]))
      return false;
  }

  return true;
}

768
static void parse_args(int argc, char *argv[], SArguments *arguments) {
769 770 771 772 773

  for (int i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-f") == 0) {
      arguments->metaFile = argv[++i];
    } else if (strcmp(argv[i], "-c") == 0) {
H
Haojun Liao 已提交
774 775 776 777 778
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-c need a valid path following!\n");
        exit(EXIT_FAILURE);
      }
779
      tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN);
780
    } else if (strcmp(argv[i], "-h") == 0) {
H
Haojun Liao 已提交
781 782 783 784 785
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-h need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
786 787
      arguments->host = argv[++i];
    } else if (strcmp(argv[i], "-p") == 0) {
788
      if ((argc == i+1) ||
789
        (!isStringNumber(argv[i+1]))) {
790 791 792 793
        printHelp();
        errorPrint("%s", "\n\t-p need a number following!\n");
        exit(EXIT_FAILURE);
      }
794
      arguments->port = atoi(argv[++i]);
795 796 797 798 799 800 801 802
    } else if (strcmp(argv[i], "-I") == 0) {
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-I need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
      ++i;
      if (0 == strcasecmp(argv[i], "taosc")) {
S
Shuduo Sang 已提交
803
          arguments->iface = TAOSC_IFACE;
804
      } else if (0 == strcasecmp(argv[i], "rest")) {
S
Shuduo Sang 已提交
805
          arguments->iface = REST_IFACE;
806
#if STMT_IFACE_ENABLED == 1
807
      } else if (0 == strcasecmp(argv[i], "stmt")) {
S
Shuduo Sang 已提交
808
          arguments->iface = STMT_IFACE;
809
#endif
810 811 812 813
      } else {
        errorPrint("%s", "\n\t-I need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
814
    } else if (strcmp(argv[i], "-u") == 0) {
H
Haojun Liao 已提交
815 816 817 818 819
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-u need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
820 821
      arguments->user = argv[++i];
    } else if (strcmp(argv[i], "-P") == 0) {
H
Haojun Liao 已提交
822 823 824 825 826
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-P need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
827 828
      arguments->password = argv[++i];
    } else if (strcmp(argv[i], "-o") == 0) {
H
Haojun Liao 已提交
829 830 831 832 833
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-o need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
834 835
      arguments->output_file = argv[++i];
    } else if (strcmp(argv[i], "-s") == 0) {
H
Haojun Liao 已提交
836 837 838 839 840
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-s need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
841 842
      arguments->sqlFile = argv[++i];
    } else if (strcmp(argv[i], "-q") == 0) {
843 844
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
845
        printHelp();
846
        errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, not-0: ASYNC. Default is SYNC.\n");
847 848
        exit(EXIT_FAILURE);
      }
849
      arguments->async_mode = atoi(argv[++i]);
850
    } else if (strcmp(argv[i], "-T") == 0) {
851 852
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
853 854 855 856
        printHelp();
        errorPrint("%s", "\n\t-T need a number following!\n");
        exit(EXIT_FAILURE);
      }
857 858
      arguments->num_of_threads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
859 860
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
861 862 863 864
        printHelp();
        errorPrint("%s", "\n\t-i need a number following!\n");
        exit(EXIT_FAILURE);
      }
865
      arguments->insert_interval = atoi(argv[++i]);
866
    } else if (strcmp(argv[i], "-qt") == 0) {
867
      if ((argc == i+1)
868
              || (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
869
        printHelp();
870
        errorPrint("%s", "\n\t-qt need a number following!\n");
H
Haojun Liao 已提交
871 872
        exit(EXIT_FAILURE);
      }
873
      arguments->query_times = atoi(argv[++i]);
874
    } else if (strcmp(argv[i], "-B") == 0) {
875 876
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
877 878 879 880
        printHelp();
        errorPrint("%s", "\n\t-B need a number following!\n");
        exit(EXIT_FAILURE);
      }
881
      arguments->interlace_rows = atoi(argv[++i]);
882
    } else if (strcmp(argv[i], "-r") == 0) {
883 884
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
885 886 887 888
        printHelp();
        errorPrint("%s", "\n\t-r need a number following!\n");
        exit(EXIT_FAILURE);
      }
889 890
      arguments->num_of_RPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-t") == 0) {
891 892 893 894 895 896
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-t need a number following!\n");
        exit(EXIT_FAILURE);
      }
897 898
      arguments->num_of_tables = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
899 900 901 902 903 904
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-n need a number following!\n");
        exit(EXIT_FAILURE);
      }
905 906
      arguments->num_of_DPT = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-d") == 0) {
H
Haojun Liao 已提交
907 908 909 910 911
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-d need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
912 913
      arguments->database = argv[++i];
    } else if (strcmp(argv[i], "-l") == 0) {
914 915 916 917 918 919
      if (argc == i+1) {
        if (!isStringNumber(argv[i+1])) {
            printHelp();
            errorPrint("%s", "\n\t-l need a number following!\n");
            exit(EXIT_FAILURE);
        }
920
      }
921
      arguments->num_of_CPR = atoi(argv[++i]);
922 923 924 925 926 927 928 929 930 931 932

      if (arguments->num_of_CPR > MAX_NUM_DATATYPE) {
          printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE);
          prompt();
          arguments->num_of_CPR = MAX_NUM_DATATYPE;
      }

      for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) {
          arguments->datatype[col] = NULL;
      }

933
    } else if (strcmp(argv[i], "-b") == 0) {
934 935 936 937 938
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-b need valid string following!\n");
        exit(EXIT_FAILURE);
      }
939 940
      ++i;
      if (strstr(argv[i], ",") == NULL) {
941
        // only one col
942
        if (strcasecmp(argv[i], "INT")
943 944 945 946 947 948 949
                && strcasecmp(argv[i], "FLOAT")
                && strcasecmp(argv[i], "TINYINT")
                && strcasecmp(argv[i], "BOOL")
                && strcasecmp(argv[i], "SMALLINT")
                && strcasecmp(argv[i], "BIGINT")
                && strcasecmp(argv[i], "DOUBLE")
                && strcasecmp(argv[i], "BINARY")
950
                && strcasecmp(argv[i], "TIMESTAMP")
951
                && strcasecmp(argv[i], "NCHAR")) {
952
          printHelp();
953
          errorPrint("%s", "-b: Invalid data_type!\n");
954
          exit(EXIT_FAILURE);
955
        }
956
        arguments->datatype[0] = argv[i];
957
      } else {
958
        // more than one col
959
        int index = 0;
960 961
        g_dupstr = strdup(argv[i]);
        char *running = g_dupstr;
962
        char *token = strsep(&running, ",");
963
        while(token != NULL) {
964
          if (strcasecmp(token, "INT")
965 966 967 968 969 970 971
                  && strcasecmp(token, "FLOAT")
                  && strcasecmp(token, "TINYINT")
                  && strcasecmp(token, "BOOL")
                  && strcasecmp(token, "SMALLINT")
                  && strcasecmp(token, "BIGINT")
                  && strcasecmp(token, "DOUBLE")
                  && strcasecmp(token, "BINARY")
972
                  && strcasecmp(token, "TIMESTAMP")
973
                  && strcasecmp(token, "NCHAR")) {
974
            printHelp();
975
            free(g_dupstr);
976
            errorPrint("%s", "-b: Invalid data_type!\n");
977
            exit(EXIT_FAILURE);
H
hzcheng 已提交
978
          }
979
          arguments->datatype[index++] = token;
980 981
          token = strsep(&running, ",");
          if (index >= MAX_NUM_DATATYPE) break;
S
TD-1057  
Shengliang Guan 已提交
982
        }
983
        arguments->datatype[index] = NULL;
984
      }
985
    } else if (strcmp(argv[i], "-w") == 0) {
986
      if ((argc == i+1) ||
987
        (!isStringNumber(argv[i+1]))) {
988 989 990 991
        printHelp();
        errorPrint("%s", "\n\t-w need a number following!\n");
        exit(EXIT_FAILURE);
      }
992 993
      arguments->len_of_binary = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-m") == 0) {
H
Haojun Liao 已提交
994
      if ((argc == i+1) ||
995
        (isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
996
        printHelp();
997
        errorPrint("%s", "\n\t-m need a letter-initial string following!\n");
H
Haojun Liao 已提交
998 999
        exit(EXIT_FAILURE);
      }
1000
      arguments->tb_prefix = argv[++i];
1001 1002
    } else if (strcmp(argv[i], "-N") == 0) {
      arguments->use_metric = false;
1003
    } else if (strcmp(argv[i], "-x") == 0) {
1004
      arguments->insert_only = false;
1005
    } else if (strcmp(argv[i], "-y") == 0) {
1006
      arguments->answer_yes = true;
1007 1008
    } else if (strcmp(argv[i], "-g") == 0) {
      arguments->debug_print = true;
1009 1010
    } else if (strcmp(argv[i], "-gg") == 0) {
      arguments->verbose_print = true;
1011 1012
    } else if (strcmp(argv[i], "-pp") == 0) {
      arguments->performance_print = true;
1013
    } else if (strcmp(argv[i], "-O") == 0) {
1014 1015 1016 1017 1018 1019
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-O need a number following!\n");
        exit(EXIT_FAILURE);
      }
1020

1021
      arguments->disorderRatio = atoi(argv[++i]);
1022

1023
      if (arguments->disorderRatio > 50) {
1024
        arguments->disorderRatio = 50;
1025
      }
1026

1027
      if (arguments->disorderRatio < 0) {
1028
        arguments->disorderRatio = 0;
1029
      }
1030

1031
    } else if (strcmp(argv[i], "-R") == 0) {
1032 1033 1034 1035 1036 1037
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-R need a number following!\n");
        exit(EXIT_FAILURE);
      }
1038

1039
      arguments->disorderRange = atoi(argv[++i]);
1040 1041 1042
      if (arguments->disorderRange < 0)
        arguments->disorderRange = 1000;

1043
    } else if (strcmp(argv[i], "-a") == 0) {
1044 1045 1046 1047 1048 1049
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-a need a number following!\n");
        exit(EXIT_FAILURE);
      }
1050 1051 1052
      arguments->replica = atoi(argv[++i]);
      if (arguments->replica > 3 || arguments->replica < 1) {
          arguments->replica = 1;
1053
      }
1054 1055
    } else if (strcmp(argv[i], "-D") == 0) {
      arguments->method_of_delete = atoi(argv[++i]);
1056 1057 1058
      if (arguments->method_of_delete > 3) {
        errorPrint("%s", "\n\t-D need a valud (0~3) number following!\n");
        exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1059
      }
1060 1061 1062 1063
    } else if ((strcmp(argv[i], "--version") == 0) ||
        (strcmp(argv[i], "-V") == 0)){
      printVersion();
      exit(0);
1064 1065 1066 1067 1068
    } else if (strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else {
      printHelp();
1069
      errorPrint("%s", "ERROR: wrong options\n");
1070
      exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1071 1072
    }
  }
1073

1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086
  int columnCount;
  for (columnCount = 0; columnCount < MAX_NUM_DATATYPE; columnCount ++) {
    if (g_args.datatype[columnCount] == NULL) {
      break;
    }
  }

  if (0 == columnCount) {
    perror("data type error!");
    exit(-1);
  }
  g_args.num_of_CPR = columnCount;

1087
  if (((arguments->debug_print) && (arguments->metaFile == NULL))
1088
          || arguments->verbose_print) {
1089
    printf("###################################################################\n");
1090
    printf("# meta file:                         %s\n", arguments->metaFile);
1091
    printf("# Server IP:                         %s:%hu\n",
1092 1093
            arguments->host == NULL ? "localhost" : arguments->host,
            arguments->port );
1094 1095
    printf("# User:                              %s\n", arguments->user);
    printf("# Password:                          %s\n", arguments->password);
1096 1097
    printf("# Use metric:                        %s\n",
            arguments->use_metric ? "true" : "false");
1098
    if (*(arguments->datatype)) {
1099 1100 1101 1102 1103 1104 1105
        printf("# Specified data type:               ");
        for (int i = 0; i < MAX_NUM_DATATYPE; i++)
            if (arguments->datatype[i])
               printf("%s,", arguments->datatype[i]);
            else
                break;
        printf("\n");
1106
    }
1107
    printf("# Insertion interval:                %"PRIu64"\n",
1108
            arguments->insert_interval);
1109
    printf("# Number of records per req:         %u\n",
1110
            arguments->num_of_RPR);
1111
    printf("# Max SQL length:                    %"PRIu64"\n",
1112
            arguments->max_sql_len);
1113
    printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
1114
    printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
1115
    printf("# Number of Tables:                  %"PRId64"\n",
1116
            arguments->num_of_tables);
1117
    printf("# Number of Data per Table:          %"PRId64"\n",
1118
            arguments->num_of_DPT);
1119 1120 1121 1122 1123
    printf("# Database name:                     %s\n", arguments->database);
    printf("# Table prefix:                      %s\n", arguments->tb_prefix);
    if (arguments->disorderRatio) {
      printf("# Data order:                        %d\n", arguments->disorderRatio);
      printf("# Data out of order rate:            %d\n", arguments->disorderRange);
1124

1125 1126 1127 1128
    }
    printf("# Delete method:                     %d\n", arguments->method_of_delete);
    printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
    printf("# Print debug info:                  %d\n", arguments->debug_print);
1129
    printf("# Print verbose info:                %d\n", arguments->verbose_print);
1130
    printf("###################################################################\n");
1131 1132

    prompt();
1133
  }
1134
}
H
hzcheng 已提交
1135

1136 1137
static bool getInfoFromJsonFile(char* file);
static void init_rand_data();
1138
static void tmfclose(FILE *fp) {
1139 1140 1141 1142
  if (NULL != fp) {
    fclose(fp);
  }
}
1143

1144
static void tmfree(char *buf) {
1145 1146
  if (NULL != buf) {
    free(buf);
H
Hui Li 已提交
1147
  }
1148
}
H
Hui Li 已提交
1149

1150
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
1151 1152 1153
  int i;
  TAOS_RES *res = NULL;
  int32_t   code = -1;
1154

S
Shuduo Sang 已提交
1155
  for (i = 0; i < 5 /* retry */; i++) {
1156 1157 1158
    if (NULL != res) {
      taos_free_result(res);
      res = NULL;
H
hzcheng 已提交
1159
    }
1160

1161 1162 1163 1164
    res = taos_query(taos, command);
    code = taos_errno(res);
    if (0 == code) {
      break;
1165
    }
H
hzcheng 已提交
1166 1167
  }

1168
  verbosePrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
1169
  if (code != 0) {
1170
    if (!quiet) {
1171 1172
      errorPrint("Failed to execute %s, reason: %s\n",
              command, taos_errstr(res));
1173
    }
1174 1175 1176
    taos_free_result(res);
    //taos_close(taos);
    return -1;
1177
  }
H
hzcheng 已提交
1178

1179 1180 1181 1182
  if (INSERT_TYPE == type) {
    int affectedRows = taos_affected_rows(res);
    taos_free_result(res);
    return affectedRows;
H
hzcheng 已提交
1183
  }
1184

1185
  taos_free_result(res);
1186 1187
  return 0;
}
H
hzcheng 已提交
1188

1189
static void appendResultBufToFile(char *resultBuf, threadInfo *pThreadInfo)
1190
{
1191 1192
  pThreadInfo->fp = fopen(pThreadInfo->filePath, "at");
  if (pThreadInfo->fp == NULL) {
1193 1194
      errorPrint(
              "%s() LN%d, failed to open result file: %s, result will not save to file\n",
1195
              __func__, __LINE__, pThreadInfo->filePath);
1196 1197 1198
      return;
  }

1199 1200 1201
  fprintf(pThreadInfo->fp, "%s", resultBuf);
  tmfclose(pThreadInfo->fp);
  pThreadInfo->fp = NULL;
1202 1203
}

1204
static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) {
1205 1206 1207 1208 1209
  TAOS_ROW    row = NULL;
  int         num_rows = 0;
  int         num_fields = taos_field_count(res);
  TAOS_FIELD *fields     = taos_fetch_fields(res);

1210 1211
  char* databuf = (char*) calloc(1, 100*1024*1024);
  if (databuf == NULL) {
1212 1213
    errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n",
            __func__, __LINE__);
1214 1215
    return ;
  }
H
hzcheng 已提交
1216

1217
  int64_t   totalLen = 0;
H
hzcheng 已提交
1218

1219
  // fetch the records row by row
1220
  while((row = taos_fetch_row(res))) {
1221 1222 1223
    if (totalLen >= 100*1024*1024 - 32000) {
        if (strlen(pThreadInfo->filePath) > 0)
            appendResultBufToFile(databuf, pThreadInfo);
1224 1225
        totalLen = 0;
        memset(databuf, 0, 100*1024*1024);
H
hzcheng 已提交
1226
    }
1227
    num_rows++;
1228
    char  temp[16000] = {0};
1229 1230 1231 1232 1233
    int len = taos_print_row(temp, row, fields, num_fields);
    len += sprintf(temp + len, "\n");
    //printf("query result:%s\n", temp);
    memcpy(databuf + totalLen, temp, len);
    totalLen += len;
1234
    verbosePrint("%s() LN%d, totalLen: %"PRId64"\n", __func__, __LINE__, totalLen);
H
hzcheng 已提交
1235 1236
  }

1237
  verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n",
1238 1239 1240
          __func__, __LINE__, databuf, pThreadInfo->filePath);
  if (strlen(pThreadInfo->filePath) > 0) {
    appendResultBufToFile(databuf, pThreadInfo);
1241
  }
1242 1243
  free(databuf);
}
H
hzcheng 已提交
1244

1245
static void selectAndGetResult(
1246 1247
        threadInfo *pThreadInfo, char *command)
{
1248 1249 1250 1251 1252 1253 1254 1255 1256
  if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) {
    TAOS_RES *res = taos_query(pThreadInfo->taos, command);
    if (res == NULL || taos_errno(res) != 0) {
        errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n",
            __func__, __LINE__, command, taos_errstr(res));
        taos_free_result(res);
        return;
    }

1257
    fetchResult(res, pThreadInfo);
1258
    taos_free_result(res);
1259

1260 1261 1262
  } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
      int retCode = postProceSql(
              g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port,
1263
              command,
1264
              pThreadInfo);
1265 1266 1267 1268 1269 1270 1271 1272
      if (0 != retCode) {
        printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID);
      }

  } else {
      errorPrint("%s() LN%d, unknown query mode: %s\n",
        __func__, __LINE__, g_queryInfo.queryMode);
  }
1273
}
H
hzcheng 已提交
1274

1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313
static int32_t rand_bool(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 2;
}

static int32_t rand_tinyint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 128;
}

static int32_t rand_smallint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 32767;
}

static int32_t rand_int(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor];
}

static int64_t rand_bigint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randbigint[cursor];
}

static float rand_float(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
1314
  return randfloat[cursor];
1315 1316
}

1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333
#if 0
static const char charNum[] = "0123456789";

static void nonrand_string(char *, int) __attribute__ ((unused));   // reserve for debugging purpose
static void nonrand_string(char *str, int size)
{
  str[0] = 0;
  if (size > 0) {
    int n;
    for (n = 0; n < size; n++) {
      str[n] = charNum[n % 10];
    }
    str[n] = 0;
  }
}
#endif

1334
static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1335

1336
static void rand_string(char *str, int size) {
1337 1338 1339 1340
  str[0] = 0;
  if (size > 0) {
    //--size;
    int n;
1341
    for (n = 0; n < size; n++) {
1342
      int key = abs(rand_tinyint()) % (int)(sizeof(charset) - 1);
1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358
      str[n] = charset[key];
    }
    str[n] = 0;
  }
}

static double rand_double() {
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randdouble[cursor];

}

static void init_rand_data() {
  for (int i = 0; i < MAX_PREPARED_RAND; i++){
1359 1360 1361 1362
    randint[i] = (int)(taosRandom() % 65535);
    randbigint[i] = (int64_t)(taosRandom() % 2147483648);
    randfloat[i] = (float)(taosRandom() / 1000.0);
    randdouble[i] = (double)(taosRandom() / 1000000.0);
1363 1364 1365
  }
}

1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1386
static int printfInsertMeta() {
1387
    SHOW_PARSE_RESULT_START();
1388

1389
  printf("interface:                  \033[33m%s\033[0m\n",
S
Shuduo Sang 已提交
1390
          (g_args.iface==TAOSC_IFACE)?"taosc":(g_args.iface==REST_IFACE)?"rest":"stmt");
1391 1392
  printf("host:                       \033[33m%s:%u\033[0m\n",
          g_Dbs.host, g_Dbs.port);
1393 1394
  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
1395
  printf("configDir:                  \033[33m%s\033[0m\n", configDir);
1396 1397
  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
1398 1399
  printf("thread num of create table: \033[33m%d\033[0m\n",
          g_Dbs.threadCountByCreateTbl);
1400
  printf("top insert interval:        \033[33m%"PRIu64"\033[0m\n",
1401
          g_args.insert_interval);
1402
  printf("number of records per req:  \033[33m%u\033[0m\n",
1403
          g_args.num_of_RPR);
1404
  printf("max sql length:             \033[33m%"PRIu64"\033[0m\n",
1405
          g_args.max_sql_len);
1406 1407

  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
1408

1409 1410
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    printf("database[\033[33m%d\033[0m]:\n", i);
1411 1412
    printf("  database[%d] name:      \033[33m%s\033[0m\n",
            i, g_Dbs.db[i].dbName);
1413
    if (0 == g_Dbs.db[i].drop) {
1414 1415 1416
      printf("  drop:                  \033[33mno\033[0m\n");
    } else {
      printf("  drop:                  \033[33myes\033[0m\n");
1417 1418 1419
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
1420 1421
      printf("  blocks:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.blocks);
1422 1423
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
1424 1425
      printf("  cache:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.cache);
1426 1427
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
1428 1429
      printf("  days:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.days);
1430 1431
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
1432 1433
      printf("  keep:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.keep);
1434 1435
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
1436 1437
      printf("  replica:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.replica);
1438 1439
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
1440 1441
      printf("  update:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.update);
1442 1443
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
1444 1445
      printf("  minRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.minRows);
1446 1447
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
1448 1449
      printf("  maxRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.maxRows);
1450 1451 1452 1453 1454
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
1455 1456
      printf("  walLevel:              \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.walLevel);
1457 1458
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
1459 1460
      printf("  fsync:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.fsync);
1461 1462
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
1463 1464
      printf("  quorum:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.quorum);
1465 1466
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1467
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
1468
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1469
        printf("  precision:             \033[33m%s\033[0m\n",
1470
            g_Dbs.db[i].dbCfg.precision);
1471
      } else {
1472
        printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
1473
                g_Dbs.db[i].dbCfg.precision);
1474
        return -1;
1475 1476 1477
      }
    }

1478
    printf("  super table count:     \033[33m%"PRIu64"\033[0m\n",
1479
        g_Dbs.db[i].superTblCount);
1480 1481
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      printf("  super table[\033[33m%"PRIu64"\033[0m]:\n", j);
1482

1483
      printf("      stbName:           \033[33m%s\033[0m\n",
1484
          g_Dbs.db[i].superTbls[j].sTblName);
1485 1486 1487

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
1488 1489
      } else if (AUTO_CREATE_SUBTBL ==
              g_Dbs.db[i].superTbls[j].autoCreateTable) {
1490 1491 1492 1493
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
      }
1494

1495 1496 1497 1498 1499 1500 1501
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
      }
1502

1503
      printf("      childTblCount:     \033[33m%"PRId64"\033[0m\n",
1504
              g_Dbs.db[i].superTbls[j].childTblCount);
1505
      printf("      childTblPrefix:    \033[33m%s\033[0m\n",
1506
              g_Dbs.db[i].superTbls[j].childTblPrefix);
1507
      printf("      dataSource:        \033[33m%s\033[0m\n",
1508
              g_Dbs.db[i].superTbls[j].dataSource);
1509 1510 1511
      printf("      iface:             \033[33m%s\033[0m\n",
          (g_Dbs.db[i].superTbls[j].iface==TAOSC_IFACE)?"taosc":
          (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest":"stmt");
1512
      if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
1513
        printf("      childTblLimit:     \033[33m%"PRId64"\033[0m\n",
1514
                g_Dbs.db[i].superTbls[j].childTblLimit);
1515
      }
1516
      if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) {
1517
        printf("      childTblOffset:    \033[33m%"PRIu64"\033[0m\n",
1518
                g_Dbs.db[i].superTbls[j].childTblOffset);
1519
      }
1520
      printf("      insertRows:        \033[33m%"PRId64"\033[0m\n",
1521
              g_Dbs.db[i].superTbls[j].insertRows);
1522
/*
1523
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1524
        printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
1525
      }else {
1526
        printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
1527
      }
1528
      */
1529
      printf("      interlaceRows:     \033[33m%u\033[0m\n",
1530
              g_Dbs.db[i].superTbls[j].interlaceRows);
1531 1532

      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1533
        printf("      stable insert interval:   \033[33m%"PRIu64"\033[0m\n",
1534
            g_Dbs.db[i].superTbls[j].insertInterval);
1535 1536
      }

1537
      printf("      disorderRange:     \033[33m%d\033[0m\n",
1538
              g_Dbs.db[i].superTbls[j].disorderRange);
1539
      printf("      disorderRatio:     \033[33m%d\033[0m\n",
1540
              g_Dbs.db[i].superTbls[j].disorderRatio);
1541
      printf("      maxSqlLen:         \033[33m%"PRIu64"\033[0m\n",
1542
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1543
      printf("      timeStampStep:     \033[33m%"PRId64"\033[0m\n",
1544
              g_Dbs.db[i].superTbls[j].timeStampStep);
1545
      printf("      startTimestamp:    \033[33m%s\033[0m\n",
1546
              g_Dbs.db[i].superTbls[j].startTimestamp);
1547
      printf("      sampleFormat:      \033[33m%s\033[0m\n",
1548
              g_Dbs.db[i].superTbls[j].sampleFormat);
1549
      printf("      sampleFile:        \033[33m%s\033[0m\n",
1550
              g_Dbs.db[i].superTbls[j].sampleFile);
1551
      printf("      tagsFile:          \033[33m%s\033[0m\n",
1552
              g_Dbs.db[i].superTbls[j].tagsFile);
1553
      printf("      columnCount:       \033[33m%d\033[0m\n",
1554
              g_Dbs.db[i].superTbls[j].columnCount);
1555 1556
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1557
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
1558 1559 1560
                       "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "nchar", 5))) {
1561
          printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
1562 1563
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1564
        } else {
1565
          printf("column[%d]:\033[33m%s\033[0m ", k,
1566
                  g_Dbs.db[i].superTbls[j].columns[k].dataType);
1567 1568 1569
        }
      }
      printf("\n");
1570 1571

      printf("      tagCount:            \033[33m%d\033[0m\n        ",
1572
              g_Dbs.db[i].superTbls[j].tagCount);
1573 1574
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1575
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
1576 1577 1578
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1579
          printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
1580 1581
                  g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1582
        } else {
1583
          printf("tag[%d]:\033[33m%s\033[0m ", k,
1584
                  g_Dbs.db[i].superTbls[j].tags[k].dataType);
1585
        }
1586 1587 1588 1589 1590
      }
      printf("\n");
    }
    printf("\n");
  }
1591 1592

  SHOW_PARSE_RESULT_END();
1593 1594

  return 0;
1595 1596 1597
}

static void printfInsertMetaToFile(FILE* fp) {
1598 1599

  SHOW_PARSE_RESULT_START_TO_FILE(fp);
1600

1601 1602
  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
  fprintf(fp, "user:                       %s\n", g_Dbs.user);
1603
  fprintf(fp, "configDir:                  %s\n", configDir);
1604 1605 1606
  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
1607
  fprintf(fp, "number of records per req:  %u\n", g_args.num_of_RPR);
1608
  fprintf(fp, "max sql length:             %"PRIu64"\n", g_args.max_sql_len);
1609
  fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
1610

1611 1612
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    fprintf(fp, "database[%d]:\n", i);
1613
    fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
1614
    if (0 == g_Dbs.db[i].drop) {
1615
      fprintf(fp, "  drop:                  no\n");
1616
    }else {
1617
      fprintf(fp, "  drop:                  yes\n");
1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1657
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
1658
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1659 1660
        fprintf(fp, "  precision:             %s\n",
                g_Dbs.db[i].dbCfg.precision);
1661
      } else {
1662 1663
        fprintf(fp, "  precision error:       %s\n",
                g_Dbs.db[i].dbCfg.precision);
1664 1665 1666
      }
    }

1667 1668
    fprintf(fp, "  super table count:     %"PRIu64"\n",
            g_Dbs.db[i].superTblCount);
1669 1670
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      fprintf(fp, "  super table[%d]:\n", j);
1671

1672 1673
      fprintf(fp, "      stbName:           %s\n",
              g_Dbs.db[i].superTbls[j].sTblName);
1674 1675 1676

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "no");
1677 1678
      } else if (AUTO_CREATE_SUBTBL
              == g_Dbs.db[i].superTbls[j].autoCreateTable) {
1679 1680 1681 1682
        fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
      } else {
        fprintf(fp, "      autoCreateTable:   %s\n",  "error");
      }
1683

1684 1685
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "no");
1686 1687
      } else if (TBL_ALREADY_EXISTS
              == g_Dbs.db[i].superTbls[j].childTblExists) {
1688 1689 1690 1691
        fprintf(fp, "      childTblExists:    %s\n",  "yes");
      } else {
        fprintf(fp, "      childTblExists:    %s\n",  "error");
      }
1692

1693
      fprintf(fp, "      childTblCount:     %"PRId64"\n",
1694 1695 1696 1697 1698
              g_Dbs.db[i].superTbls[j].childTblCount);
      fprintf(fp, "      childTblPrefix:    %s\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      fprintf(fp, "      dataSource:        %s\n",
              g_Dbs.db[i].superTbls[j].dataSource);
1699 1700 1701
      fprintf(fp, "      iface:             %s\n",
          (g_Dbs.db[i].superTbls[j].iface==TAOSC_IFACE)?"taosc":
          (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest":"stmt");
1702
      fprintf(fp, "      insertRows:        %"PRId64"\n",
1703
              g_Dbs.db[i].superTbls[j].insertRows);
1704
      fprintf(fp, "      interlace rows:    %u\n",
1705
              g_Dbs.db[i].superTbls[j].interlaceRows);
1706
      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1707
        fprintf(fp, "      stable insert interval:   %"PRIu64"\n",
1708
                g_Dbs.db[i].superTbls[j].insertInterval);
1709
      }
1710
/*
1711
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1712
        fprintf(fp, "      multiThreadWriteOneTbl:  no\n");
1713
      }else {
1714
        fprintf(fp, "      multiThreadWriteOneTbl:  yes\n");
1715
      }
1716
      */
1717
      fprintf(fp, "      interlaceRows:     %u\n",
1718
              g_Dbs.db[i].superTbls[j].interlaceRows);
1719 1720 1721 1722
      fprintf(fp, "      disorderRange:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      fprintf(fp, "      disorderRatio:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
1723
      fprintf(fp, "      maxSqlLen:         %"PRIu64"\n",
1724
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1725

1726 1727 1728 1729
      fprintf(fp, "      timeStampStep:     %"PRId64"\n",
              g_Dbs.db[i].superTbls[j].timeStampStep);
      fprintf(fp, "      startTimestamp:    %s\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
1730 1731 1732 1733 1734 1735
      fprintf(fp, "      sampleFormat:      %s\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      fprintf(fp, "      sampleFile:        %s\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      fprintf(fp, "      tagsFile:          %s\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
1736

1737 1738
      fprintf(fp, "      columnCount:       %d\n        ",
              g_Dbs.db[i].superTbls[j].columnCount);
1739 1740
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1741
        if ((0 == strncasecmp(
1742 1743
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "binary", strlen("binary")))
1744 1745
                || (0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
1746
                        "nchar", strlen("nchar")))) {
1747 1748 1749
          fprintf(fp, "column[%d]:%s(%d) ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1750
        } else {
1751 1752
          fprintf(fp, "column[%d]:%s ",
                  k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
1753 1754 1755
        }
      }
      fprintf(fp, "\n");
1756 1757 1758

      fprintf(fp, "      tagCount:            %d\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1759 1760
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1761
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
1762 1763 1764
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1765 1766
          fprintf(fp, "tag[%d]:%s(%d) ",
                  k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
1767
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1768 1769
        } else {
          fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
1770
        }
1771 1772 1773 1774 1775
      }
      fprintf(fp, "\n");
    }
    fprintf(fp, "\n");
  }
1776

1777
  SHOW_PARSE_RESULT_END_TO_FILE(fp);
1778 1779 1780
}

static void printfQueryMeta() {
1781

1782
  SHOW_PARSE_RESULT_START();
1783

1784
  printf("host:                    \033[33m%s:%u\033[0m\n",
1785
          g_queryInfo.host, g_queryInfo.port);
1786 1787 1788 1789
  printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
  printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

  printf("\n");
1790 1791 1792

  if ((SUBSCRIBE_TEST == g_args.test_mode) || (QUERY_TEST == g_args.test_mode)) {
    printf("specified table query info:                   \n");
1793
    printf("sqlCount:       \033[33m%d\033[0m\n",
1794
      g_queryInfo.specifiedQueryInfo.sqlCount);
1795 1796 1797
    if (g_queryInfo.specifiedQueryInfo.sqlCount > 0) {
      printf("specified tbl query times:\n");
      printf("                \033[33m%"PRIu64"\033[0m\n",
1798
      g_queryInfo.specifiedQueryInfo.queryTimes);
1799 1800 1801
      printf("query interval: \033[33m%"PRIu64" ms\033[0m\n",
        g_queryInfo.specifiedQueryInfo.queryInterval);
      printf("top query times:\033[33m%"PRIu64"\033[0m\n", g_args.query_times);
1802
      printf("concurrent:     \033[33m%d\033[0m\n",
1803 1804
      g_queryInfo.specifiedQueryInfo.concurrent);
      printf("mod:            \033[33m%s\033[0m\n",
1805
        (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync");
1806
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1807
        g_queryInfo.specifiedQueryInfo.subscribeInterval);
1808
      printf("restart:        \033[33m%d\033[0m\n",
1809
        g_queryInfo.specifiedQueryInfo.subscribeRestart);
1810
      printf("keepProgress:   \033[33m%d\033[0m\n",
1811
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
1812

1813 1814
      for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        printf("  sql[%d]: \033[33m%s\033[0m\n",
1815 1816 1817 1818 1819 1820
            i, g_queryInfo.specifiedQueryInfo.sql[i]);
      }
      printf("\n");
    }

    printf("super table query info:\n");
1821
    printf("sqlCount:       \033[33m%d\033[0m\n",
1822 1823 1824 1825 1826 1827 1828
      g_queryInfo.superQueryInfo.sqlCount);

    if (g_queryInfo.superQueryInfo.sqlCount > 0) {
      printf("query interval: \033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryInterval);
      printf("threadCnt:      \033[33m%d\033[0m\n",
        g_queryInfo.superQueryInfo.threadCnt);
1829
      printf("childTblCount:  \033[33m%"PRId64"\033[0m\n",
1830 1831 1832 1833 1834 1835 1836
        g_queryInfo.superQueryInfo.childTblCount);
      printf("stable name:    \033[33m%s\033[0m\n",
        g_queryInfo.superQueryInfo.sTblName);
      printf("stb query times:\033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryTimes);

      printf("mod:            \033[33m%s\033[0m\n",
1837
        (g_queryInfo.superQueryInfo.asyncMode)?"async":"sync");
1838
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1839
        g_queryInfo.superQueryInfo.subscribeInterval);
1840
      printf("restart:        \033[33m%d\033[0m\n",
1841
        g_queryInfo.superQueryInfo.subscribeRestart);
1842
      printf("keepProgress:   \033[33m%d\033[0m\n",
1843
        g_queryInfo.superQueryInfo.subscribeKeepProgress);
1844

1845 1846 1847 1848 1849 1850
      for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
        printf("  sql[%d]: \033[33m%s\033[0m\n",
            i, g_queryInfo.superQueryInfo.sql[i]);
      }
      printf("\n");
    }
1851
  }
1852

1853
  SHOW_PARSE_RESULT_END();
1854 1855
}

1856
static char* formatTimestamp(char* buf, int64_t val, int precision) {
1857
  time_t tt;
1858 1859 1860
  if (precision == TSDB_TIME_PRECISION_NANO) {
    tt = (time_t)(val / 1000000000);
  } else if (precision == TSDB_TIME_PRECISION_MICRO) {
1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880
    tt = (time_t)(val / 1000000);
  } else {
    tt = (time_t)(val / 1000);
  }

/* comment out as it make testcases like select_with_tags.sim fail.
  but in windows, this may cause the call to localtime crash if tt < 0,
  need to find a better solution.
  if (tt < 0) {
    tt = 0;
  }
  */

#ifdef WINDOWS
  if (tt < 0) tt = 0;
#endif

  struct tm* ptm = localtime(&tt);
  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);

1881 1882 1883
  if (precision == TSDB_TIME_PRECISION_NANO) {
    sprintf(buf + pos, ".%09d", (int)(val % 1000000000));
  } else if (precision == TSDB_TIME_PRECISION_MICRO) {
1884 1885 1886 1887 1888 1889 1890 1891
    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
  } else {
    sprintf(buf + pos, ".%03d", (int)(val % 1000));
  }

  return buf;
}

1892
static void xDumpFieldToFile(FILE* fp, const char* val,
1893
        TAOS_FIELD* field, int32_t length, int precision) {
1894

1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929
  if (val == NULL) {
    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
    return;
  }

  char buf[TSDB_MAX_BYTES_PER_ROW];
  switch (field->type) {
    case TSDB_DATA_TYPE_BOOL:
      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
      break;
    case TSDB_DATA_TYPE_TINYINT:
      fprintf(fp, "%d", *((int8_t *)val));
      break;
    case TSDB_DATA_TYPE_SMALLINT:
      fprintf(fp, "%d", *((int16_t *)val));
      break;
    case TSDB_DATA_TYPE_INT:
      fprintf(fp, "%d", *((int32_t *)val));
      break;
    case TSDB_DATA_TYPE_BIGINT:
      fprintf(fp, "%" PRId64, *((int64_t *)val));
      break;
    case TSDB_DATA_TYPE_FLOAT:
      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
      break;
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      memcpy(buf, val, length);
      buf[length] = 0;
      fprintf(fp, "\'%s\'", buf);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
1930
      formatTimestamp(buf, *(int64_t*)val, precision);
1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945
      fprintf(fp, "'%s'", buf);
      break;
    default:
      break;
  }
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  FILE* fp = fopen(fname, "at");
  if (fp == NULL) {
1946
    errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960
    return -1;
  }

  int num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int precision = taos_result_precision(tres);

  for (int col = 0; col < num_fields; col++) {
    if (col > 0) {
      fprintf(fp, ",");
    }
    fprintf(fp, "%s", fields[col].name);
  }
  fputc('\n', fp);
1961

1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981
  int numOfRows = 0;
  do {
    int32_t* length = taos_fetch_lengths(tres);
    for (int i = 0; i < num_fields; i++) {
      if (i > 0) {
        fputc(',', fp);
      }
      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
    }
    fputc('\n', fp);

    numOfRows++;
    row = taos_fetch_row(tres);
  } while( row != NULL);

  fclose(fp);

  return numOfRows;
}

1982 1983
static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
  TAOS_RES * res;
1984 1985
  TAOS_ROW row = NULL;
  int count = 0;
1986 1987

  res = taos_query(taos, "show databases;");
1988
  int32_t code = taos_errno(res);
1989

1990
  if (code != 0) {
1991
    errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
1992 1993 1994 1995 1996
    return -1;
  }

  TAOS_FIELD *fields = taos_fetch_fields(res);

1997
  while((row = taos_fetch_row(res)) != NULL) {
1998
    // sys database name : 'log'
1999
    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log",
2000
                fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) {
2001 2002
      continue;
    }
2003 2004 2005

    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
    if (dbInfos[count] == NULL) {
2006
      errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
2007 2008 2009
      return -1;
    }

2010
    tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
2011
            fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
2012
    formatTimestamp(dbInfos[count]->create_time,
2013 2014
            *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
            TSDB_TIME_PRECISION_MILLI);
S
Shuduo Sang 已提交
2015
    dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
2016
    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
2017 2018
    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
2019
    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
2020

2021
    tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
2022
            fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
2023 2024 2025 2026 2027 2028 2029
    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
2030
    dbInfos[count]->cachelast =
2031
      (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));
2032

2033
    tstrncpy(dbInfos[count]->precision,
2034 2035
            (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
            fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
2036
    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
2037
    tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
2038
            fields[TSDB_SHOW_DB_STATUS_INDEX].bytes);
2039

2040 2041
    count++;
    if (count > MAX_DATABASE_COUNT) {
2042
      errorPrint("%s() LN%d, The database count overflow than %d\n",
2043
         __func__, __LINE__, MAX_DATABASE_COUNT);
2044 2045 2046 2047 2048 2049 2050
      break;
    }
  }

  return count;
}

2051
static void printfDbInfoForQueryToFile(
2052
        char* filename, SDbInfo* dbInfos, int index) {
2053

2054
  if (filename[0] == 0)
2055
      return;
2056 2057 2058

  FILE *fp = fopen(filename, "at");
  if (fp == NULL) {
2059
    errorPrint( "failed to open file: %s\n", filename);
2060
    return;
2061 2062 2063 2064 2065
  }

  fprintf(fp, "================ database[%d] ================\n", index);
  fprintf(fp, "name: %s\n", dbInfos->name);
  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
S
Shuduo Sang 已提交
2066
  fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables);
2067
  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);
2068 2069
  fprintf(fp, "replica: %d\n", dbInfos->replica);
  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
2070 2071
  fprintf(fp, "days: %d\n", dbInfos->days);
  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);
2072 2073 2074 2075 2076 2077 2078
  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
  fprintf(fp, "comp: %d\n", dbInfos->comp);
2079 2080
  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);
  fprintf(fp, "precision: %s\n", dbInfos->precision);
2081
  fprintf(fp, "update: %d\n", dbInfos->update);
2082
  fprintf(fp, "status: %s\n", dbInfos->status);
2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096
  fprintf(fp, "\n");

  fclose(fp);
}

static void printfQuerySystemInfo(TAOS * taos) {
  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
  TAOS_RES* res;

  time_t t;
  struct tm* lt;
  time(&t);
  lt = localtime(&t);
2097
  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
2098 2099
          lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
          lt->tm_sec);
2100

2101 2102
  // show variables
  res = taos_query(taos, "show variables;");
2103
  //fetchResult(res, filename);
2104 2105 2106 2107 2108
  xDumpResultToFile(filename, res);

  // show dnodes
  res = taos_query(taos, "show dnodes;");
  xDumpResultToFile(filename, res);
2109
  //fetchResult(res, filename);
2110

2111 2112 2113 2114
  // show databases
  res = taos_query(taos, "show databases;");
  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
  if (dbInfos == NULL) {
2115
    errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
2116 2117 2118
    return;
  }
  int dbCount = getDbFromServer(taos, dbInfos);
2119
  if (dbCount <= 0) {
2120 2121
      free(dbInfos);
      return;
2122
  }
2123 2124

  for (int i = 0; i < dbCount; i++) {
2125
    // printf database info
2126
    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
2127

2128 2129 2130 2131
    // show db.vgroups
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);
2132

2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143
    // show db.stables
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);

    free(dbInfos[i]);
  }

  free(dbInfos);
}

2144
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
2145
        char* sqlstr, threadInfo *pThreadInfo)
2146
{
2147
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
2148

2149
    char *url = "/rest/sql";
2150

2151 2152 2153 2154
    int bytes, sent, received, req_str_len, resp_len;
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
2155

2156
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;
2157

2158 2159 2160 2161 2162
    request_buf = malloc(req_buf_len);
    if (NULL == request_buf) {
      errorPrint("%s", "ERROR, cannot allocate memory.\n");
      exit(EXIT_FAILURE);
    }
2163

2164 2165
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};
2166

2167 2168 2169 2170 2171 2172 2173 2174
    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
2175

2176 2177 2178 2179
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);
2180

2181
    char base64_buf[INPUT_BUF_LEN];
2182
#ifdef WINDOWS
2183
    WSADATA wsaData;
2184
    WSAStartup(MAKEWORD(2, 2), &wsaData);
2185 2186
    SOCKET sockfd;
#else
2187
    int sockfd;
2188
#endif
2189 2190
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (sockfd < 0) {
2191
#ifdef WINDOWS
2192
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
2193
#endif
2194 2195 2196 2197
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
        free(request_buf);
        ERROR_EXIT("ERROR opening socket");
    }
2198

2199
    int retConn = connect(sockfd, (struct sockaddr *)pServAddr, sizeof(struct sockaddr));
2200 2201 2202 2203 2204
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
        free(request_buf);
        ERROR_EXIT("ERROR connecting");
    }
2205

2206
    memset(base64_buf, 0, INPUT_BUF_LEN);
2207

2208 2209 2210 2211 2212 2213 2214 2215
    for (int n = 0, m = 0; n < userpass_buf_len;) {
      uint32_t oct_a = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t oct_b = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t oct_c = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;
2216

2217 2218 2219 2220 2221
      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }
2222

2223 2224
    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';
2225

2226 2227 2228
    debugPrint("%s() LN%d: auth string base64 encoded: %s\n",
            __func__, __LINE__, base64_buf);
    char *auth = base64_buf;
2229

2230 2231 2232 2233 2234 2235 2236 2237 2238
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
2239

2240 2241 2242
    req_str_len = strlen(request_buf);
    sent = 0;
    do {
2243
#ifdef WINDOWS
2244
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
2245
#else
2246
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
2247
#endif
2248 2249 2250 2251 2252 2253
        if (bytes < 0)
            ERROR_EXIT("ERROR writing message to socket");
        if (bytes == 0)
            break;
        sent+=bytes;
    } while(sent < req_str_len);
2254

2255 2256 2257 2258
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
    received = 0;
    do {
2259
#ifdef WINDOWS
2260
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
2261
#else
2262
        bytes = read(sockfd, response_buf + received, resp_len - received);
2263
#endif
2264 2265 2266 2267 2268 2269 2270 2271
        if (bytes < 0) {
            free(request_buf);
            ERROR_EXIT("ERROR reading response from socket");
        }
        if (bytes == 0)
            break;
        received += bytes;
    } while(received < resp_len);
2272

2273 2274 2275 2276
    if (received == resp_len) {
        free(request_buf);
        ERROR_EXIT("ERROR storing complete response from socket");
    }
H
Haojun Liao 已提交
2277

2278 2279
    response_buf[RESP_BUF_LEN - 1] = '\0';
    printf("Response:\n%s\n", response_buf);
H
Haojun Liao 已提交
2280

2281 2282
    if (strlen(pThreadInfo->filePath) > 0) {
       appendResultBufToFile(response_buf, pThreadInfo);
2283 2284
    }

2285
    free(request_buf);
2286
#ifdef WINDOWS
2287
    closesocket(sockfd);
2288 2289
    WSACleanup();
#else
2290
    close(sockfd);
2291
#endif
2292

2293
    return 0;
2294 2295
}

2296
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
2297 2298
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
2299
    errorPrint("%s() LN%d, calloc failed! size:%d\n",
2300
        __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
2301 2302
    return NULL;
  }
2303

2304
  int    dataLen = 0;
2305
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2306
          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);
2307

2308 2309 2310
  return dataBuf;
}

2311
static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) {
2312 2313 2314 2315 2316 2317 2318 2319 2320
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
    printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
    return NULL;
  }

  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
  for (int i = 0; i < stbInfo->tagCount; i++) {
2321
    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
2322
            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
2323
      if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
2324
        printf("binary or nchar length overflow, max size:%u\n",
2325
                (uint32_t)TSDB_MAX_BINARY_LEN);
2326 2327 2328
        tmfree(dataBuf);
        return NULL;
      }
2329

2330 2331
      int tagBufLen = stbInfo->tags[i].dataLen + 1;
      char* buf = (char*)calloc(tagBufLen, 1);
2332 2333 2334 2335 2336
      if (NULL == buf) {
        printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
        tmfree(dataBuf);
        return NULL;
      }
2337 2338 2339 2340 2341 2342 2343

      if (tableSeq % 2) {
        tstrncpy(buf, "beijing", tagBufLen);
      } else {
        tstrncpy(buf, "shanghai", tagBufLen);
      }
      //rand_string(buf, stbInfo->tags[i].dataLen);
2344
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2345
              "\'%s\', ", buf);
2346
      tmfree(buf);
2347
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2348
                "int", strlen("int"))) {
2349
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2350
              "%d, ", tableSeq);
2351
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2352
                "bigint", strlen("bigint"))) {
2353
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2354
              "%"PRId64", ", rand_bigint());
2355
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2356
                "float", strlen("float"))) {
2357
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2358
              "%f, ", rand_float());
2359
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2360
                "double", strlen("double"))) {
2361
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2362
              "%f, ", rand_double());
2363
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2364
                "smallint", strlen("smallint"))) {
2365
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2366
              "%d, ", rand_smallint());
2367
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2368
                "tinyint", strlen("tinyint"))) {
2369
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2370
              "%d, ", rand_tinyint());
2371
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2372
                "bool", strlen("bool"))) {
2373
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2374
              "%d, ", rand_bool());
2375
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2376
                "timestamp", strlen("timestamp"))) {
2377
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2378
              "%"PRId64", ", rand_bigint());
2379 2380 2381 2382 2383 2384
    }  else {
      printf("No support data type: %s\n", stbInfo->tags[i].dataType);
      tmfree(dataBuf);
      return NULL;
    }
  }
2385

2386
  dataLen -= 2;
2387
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");
2388 2389 2390
  return dataBuf;
}

2391
static int calcRowLen(SSuperTable*  superTbls) {
2392 2393
  int colIndex;
  int  lenOfOneRow = 0;
2394

2395 2396
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2397

2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
2414
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2415
      lenOfOneRow += 42;
2416
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
2430

2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
2447
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2448 2449 2450 2451 2452 2453 2454 2455
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2456

2457 2458 2459 2460
  return 0;
}


2461
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
2462
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
2463
        int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) {
2464

2465
  char command[BUFFER_SIZE] = "\0";
2466 2467
  char limitBuf[100] = "\0";

2468
  TAOS_RES * res;
2469 2470 2471
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2472 2473

  if (offset >= 0) {
2474
    snprintf(limitBuf, 100, " limit %"PRId64" offset %"PRIu64"",
2475
            limit, offset);
2476 2477
  }

2478
  //get all child table name use cmd: select tbname from superTblName;
2479
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s",
2480
          dbName, sTblName, limitBuf);
2481

2482
  res = taos_query(taos, command);
2483 2484 2485 2486
  int32_t code = taos_errno(res);
  if (code != 0) {
    taos_free_result(res);
    taos_close(taos);
2487
    errorPrint("%s() LN%d, failed to run command %s\n",
2488
           __func__, __LINE__, command);
2489 2490 2491
    exit(-1);
  }

2492 2493
  int64_t childTblCount = (limit < 0)?10000:limit;
  int64_t count = 0;
2494 2495 2496
  if (childTblName == NULL) {
    childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (NULL ==  childTblName) {
2497 2498 2499 2500
    taos_free_result(res);
        taos_close(taos);
        errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__);
        exit(-1);
2501 2502 2503
    }
  }

2504
  char* pTblName = childTblName;
2505
  while((row = taos_fetch_row(res)) != NULL) {
2506
    int32_t* len = taos_fetch_lengths(res);
2507 2508 2509 2510 2511 2512 2513

    if (0 == strlen((char *)row[0])) {
        errorPrint("%s() LN%d, No.%"PRId64" table return empty name\n",
            __func__, __LINE__, count);
        exit(-1);
    }

2514
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2515 2516 2517
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2518
      char *tmp = realloc(childTblName,
2519
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2520 2521 2522
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2523
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
2524
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2525 2526
      } else {
        // exit, if allocate more memory failed
2527
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
2528
               __func__, __LINE__, dbName, sTblName);
2529 2530 2531 2532 2533 2534
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2535
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2536
  }
2537

2538 2539 2540 2541 2542 2543 2544
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2545
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
2546
        char* sTblName, char** childTblNameOfSuperTbl,
2547
        int64_t* childTblCountOfSuperTbl) {
2548

2549 2550
    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
2551
            -1, 0);
2552 2553
}

2554
static int getSuperTableFromServer(TAOS * taos, char* dbName,
2555
        SSuperTable*  superTbls) {
2556

2557
  char command[BUFFER_SIZE] = "\0";
2558
  TAOS_RES * res;
2559 2560
  TAOS_ROW row = NULL;
  int count = 0;
2561

2562
  //get schema use cmd: describe superTblName;
2563
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2564
  res = taos_query(taos, command);
2565 2566 2567
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2568
    taos_free_result(res);
2569 2570 2571 2572 2573 2574
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
2575
  while((row = taos_fetch_row(res)) != NULL) {
2576 2577 2578
    if (0 == count) {
      count++;
      continue;
2579
    }
2580 2581

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2582
      tstrncpy(superTbls->tags[tagIndex].field,
2583 2584
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
2585
      tstrncpy(superTbls->tags[tagIndex].dataType,
2586
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
2587
              min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes));
2588 2589 2590
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
2591 2592
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2593
      tagIndex++;
2594
    } else {
2595
      tstrncpy(superTbls->columns[columnIndex].field,
2596 2597
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
2598
      tstrncpy(superTbls->columns[columnIndex].dataType,
2599
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
2600
              min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes));
2601 2602 2603
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
2604 2605
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2617
/*
2618
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
2619
    //get all child table name use cmd: select tbname from superTblName;
2620 2621 2622
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2623
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2624 2625
      return -1;
    }
2626
    getAllChildNameOfSuperTable(taos, dbName,
2627 2628 2629
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2630
  }
2631
  */
2632 2633 2634
  return 0;
}

H
Haojun Liao 已提交
2635
static int createSuperTable(
2636 2637
        TAOS * taos, char* dbName,
        SSuperTable*  superTbl) {
H
Haojun Liao 已提交
2638

2639
  char command[BUFFER_SIZE] = "\0";
2640

2641 2642 2643 2644 2645
  char cols[STRING_LEN] = "\0";
  int colIndex;
  int len = 0;

  int  lenOfOneRow = 0;
2646 2647 2648

  if (superTbl->columnCount == 0) {
    errorPrint("%s() LN%d, super table column count is %d\n",
2649
            __func__, __LINE__, superTbl->columnCount);
2650 2651 2652 2653 2654
    return -1;
  }

  for (colIndex = 0; colIndex < superTbl->columnCount; colIndex++) {
    char* dataType = superTbl->columns[colIndex].dataType;
2655

2656
    if (strcasecmp(dataType, "BINARY") == 0) {
2657
      len += snprintf(cols + len, STRING_LEN - len,
2658 2659
          ", col%d %s(%d)", colIndex, "BINARY",
          superTbl->columns[colIndex].dataLen);
2660
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2661
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
2662
      len += snprintf(cols + len, STRING_LEN - len,
2663 2664
          ", col%d %s(%d)", colIndex, "NCHAR",
          superTbl->columns[colIndex].dataLen);
2665
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
      lenOfOneRow += 22;
2684
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2685 2686
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
      lenOfOneRow += 42;
2687
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2688 2689 2690 2691
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
      lenOfOneRow += 21;
    } else {
      taos_close(taos);
2692
      errorPrint("%s() LN%d, config error data type : %s\n",
2693
         __func__, __LINE__, dataType);
2694 2695 2696 2697
      exit(-1);
    }
  }

2698 2699
  superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp
  //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbl[j].sTblName, g_Dbs.db[i].superTbl[j].columnCount, lenOfOneRow);
2700 2701

  // save for creating child table
2702 2703 2704
  superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1);
  if (NULL == superTbl->colsOfCreateChildTable) {
    errorPrint("%s() LN%d, Failed when calloc, size:%d",
2705
           __func__, __LINE__, len+1);
2706 2707 2708 2709
    taos_close(taos);
    exit(-1);
  }

2710
  snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
2711
  verbosePrint("%s() LN%d: %s\n",
2712
      __func__, __LINE__, superTbl->colsOfCreateChildTable);
2713

2714 2715
  if (superTbl->tagCount == 0) {
    errorPrint("%s() LN%d, super table tag count is %d\n",
2716
            __func__, __LINE__, superTbl->tagCount);
2717 2718
    return -1;
  }
2719

2720 2721 2722
  char tags[STRING_LEN] = "\0";
  int tagIndex;
  len = 0;
2723

2724 2725 2726 2727
  int lenOfTagOfOneRow = 0;
  len += snprintf(tags + len, STRING_LEN - len, "(");
  for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) {
    char* dataType = superTbl->tags[tagIndex].dataType;
2728

2729 2730
    if (strcasecmp(dataType, "BINARY") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
2731
              "BINARY", superTbl->tags[tagIndex].dataLen);
2732 2733 2734
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
2735
              "NCHAR", superTbl->tags[tagIndex].dataLen);
2736 2737 2738
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2739
              "INT");
2740 2741 2742
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2743
              "BIGINT");
2744 2745 2746
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2747
              "SMALLINT");
2748 2749 2750
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2751
              "TINYINT");
2752 2753 2754
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2755
              "BOOL");
2756 2757 2758
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2759
              "FLOAT");
2760 2761 2762
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2763
              "DOUBLE");
2764 2765 2766
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42;
    } else {
      taos_close(taos);
2767
      errorPrint("%s() LN%d, config error tag type : %s\n",
2768
         __func__, __LINE__, dataType);
2769
      exit(-1);
2770 2771
    }
  }
2772 2773 2774 2775 2776 2777 2778

  len -= 2;
  len += snprintf(tags + len, STRING_LEN - len, ")");

  superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow;

  snprintf(command, BUFFER_SIZE,
2779 2780
          "create table if not exists %s.%s (ts timestamp%s) tags %s",
          dbName, superTbl->sTblName, cols, tags);
2781
  if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2782 2783 2784
      errorPrint( "create supertable %s failed!\n\n",
              superTbl->sTblName);
      return -1;
2785 2786
  }
  debugPrint("create supertable %s success!\n\n", superTbl->sTblName);
2787 2788 2789
  return 0;
}

2790
static int createDatabasesAndStables() {
2791 2792 2793 2794
  TAOS * taos = NULL;
  int    ret = 0;
  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
  if (taos == NULL) {
2795
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2796
    return -1;
2797 2798 2799
  }
  char command[BUFFER_SIZE] = "\0";

2800
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2801 2802
    if (g_Dbs.db[i].drop) {
      sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
2803
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2804 2805 2806 2807
        taos_close(taos);
        return -1;
      }

2808
      int dataLen = 0;
2809
      dataLen += snprintf(command + dataLen,
2810
          BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2811

2812 2813
      if (g_Dbs.db[i].dbCfg.blocks > 0) {
        dataLen += snprintf(command + dataLen,
2814
            BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
2815 2816 2817
      }
      if (g_Dbs.db[i].dbCfg.cache > 0) {
        dataLen += snprintf(command + dataLen,
2818
            BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
2819 2820 2821
      }
      if (g_Dbs.db[i].dbCfg.days > 0) {
        dataLen += snprintf(command + dataLen,
2822
            BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
2823 2824 2825
      }
      if (g_Dbs.db[i].dbCfg.keep > 0) {
        dataLen += snprintf(command + dataLen,
2826
            BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
2827 2828 2829
      }
      if (g_Dbs.db[i].dbCfg.quorum > 1) {
        dataLen += snprintf(command + dataLen,
2830
            BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
2831 2832 2833
      }
      if (g_Dbs.db[i].dbCfg.replica > 0) {
        dataLen += snprintf(command + dataLen,
2834
            BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
2835 2836 2837
      }
      if (g_Dbs.db[i].dbCfg.update > 0) {
        dataLen += snprintf(command + dataLen,
2838
            BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
2839 2840 2841 2842 2843 2844 2845
      }
      //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
      //  dataLen += snprintf(command + dataLen,
      //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
      //}
      if (g_Dbs.db[i].dbCfg.minRows > 0) {
        dataLen += snprintf(command + dataLen,
2846
            BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
2847 2848 2849
      }
      if (g_Dbs.db[i].dbCfg.maxRows > 0) {
        dataLen += snprintf(command + dataLen,
2850
            BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
2851 2852 2853
      }
      if (g_Dbs.db[i].dbCfg.comp > 0) {
        dataLen += snprintf(command + dataLen,
2854
            BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
2855 2856 2857
      }
      if (g_Dbs.db[i].dbCfg.walLevel > 0) {
        dataLen += snprintf(command + dataLen,
2858
            BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
2859 2860 2861
      }
      if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
        dataLen += snprintf(command + dataLen,
2862
            BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
2863 2864 2865
      }
      if (g_Dbs.db[i].dbCfg.fsync > 0) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2866
                " fsync %d", g_Dbs.db[i].dbCfg.fsync);
2867 2868
      }
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
2869 2870
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                      "us", strlen("us")))) {
2871
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2872
                " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
2873 2874 2875 2876 2877 2878 2879 2880
      }

      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
        taos_close(taos);
        errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
        return -1;
      }
      printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
2881 2882
    }

2883
    debugPrint("%s() LN%d supertbl count:%"PRIu64"\n",
2884
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
2885 2886 2887

    int validStbCount = 0;

S
Shuduo Sang 已提交
2888
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
2889 2890 2891 2892 2893
      sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
              g_Dbs.db[i].superTbls[j].sTblName);
      ret = queryDbExec(taos, command, NO_INSERT_TYPE, true);

      if ((ret != 0) || (g_Dbs.db[i].drop)) {
2894
        ret = createSuperTable(taos, g_Dbs.db[i].dbName,
2895
                &g_Dbs.db[i].superTbls[j]);
2896

2897
        if (0 != ret) {
S
Shuduo Sang 已提交
2898
          errorPrint("create super table %"PRIu64" failed!\n\n", j);
2899
          continue;
2900 2901 2902
        }
      }

2903
      ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
2904
                &g_Dbs.db[i].superTbls[j]);
2905 2906
      if (0 != ret) {
        errorPrint("\nget super table %s.%s info failed!\n\n",
2907
                g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName);
2908
        continue;
2909
      }
2910 2911

      validStbCount ++;
2912
    }
2913 2914

    g_Dbs.db[i].superTblCount = validStbCount;
2915 2916 2917 2918 2919 2920
  }

  taos_close(taos);
  return 0;
}

2921 2922
static void* createTable(void *sarg)
{
2923 2924
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
2925

S
Shuduo Sang 已提交
2926
  uint64_t  lastPrintTime = taosGetTimestampMs();
2927

2928
  int buff_len;
2929
  buff_len = BUFFER_SIZE / 8;
2930

2931 2932
  pThreadInfo->buffer = calloc(buff_len, 1);
  if (pThreadInfo->buffer == NULL) {
2933
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
2934 2935
    exit(-1);
  }
2936 2937 2938

  int len = 0;
  int batchNum = 0;
2939

2940
  verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n",
2941 2942
          __func__, __LINE__,
          pThreadInfo->start_table_from, pThreadInfo->end_table_to);
2943

2944
  for (uint64_t i = pThreadInfo->start_table_from;
2945
          i <= pThreadInfo->end_table_to; i++) {
2946
    if (0 == g_Dbs.use_metric) {
2947
      snprintf(pThreadInfo->buffer, buff_len,
2948
              "create table if not exists %s.%s%"PRIu64" %s;",
2949 2950 2951
              pThreadInfo->db_name,
              g_args.tb_prefix, i,
              pThreadInfo->cols);
2952
    } else {
2953 2954
      if (superTblInfo == NULL) {
        errorPrint("%s() LN%d, use metric, but super table info is NULL\n",
2955
                  __func__, __LINE__);
2956
        free(pThreadInfo->buffer);
2957 2958 2959 2960
        exit(-1);
      } else {
        if (0 == len) {
          batchNum = 0;
2961 2962
          memset(pThreadInfo->buffer, 0, buff_len);
          len += snprintf(pThreadInfo->buffer + len,
2963
                  buff_len - len, "create table ");
2964 2965 2966
        }
        char* tagsValBuf = NULL;
        if (0 == superTblInfo->tagSource) {
2967
          tagsValBuf = generateTagVaulesForStb(superTblInfo, i);
2968 2969
        } else {
          tagsValBuf = getTagValueFromTagSample(
2970 2971
                  superTblInfo,
                  i % superTblInfo->tagSampleCount);
2972 2973
        }
        if (NULL == tagsValBuf) {
2974
          free(pThreadInfo->buffer);
2975 2976
          return NULL;
        }
2977
        len += snprintf(pThreadInfo->buffer + len,
2978
                buff_len - len,
2979
                "if not exists %s.%s%"PRIu64" using %s.%s tags %s ",
2980 2981 2982
                pThreadInfo->db_name, superTblInfo->childTblPrefix,
                i, pThreadInfo->db_name,
                superTblInfo->sTblName, tagsValBuf);
2983 2984 2985
        free(tagsValBuf);
        batchNum++;
        if ((batchNum < superTblInfo->batchCreateTableNum)
2986 2987
                && ((buff_len - len)
                    >= (superTblInfo->lenOfTagOfOneRow + 256))) {
2988 2989
          continue;
        }
2990 2991 2992 2993
      }
    }

    len = 0;
2994 2995 2996 2997
    if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer,
                NO_INSERT_TYPE, false)){
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer);
      free(pThreadInfo->buffer);
2998 2999 3000
      return NULL;
    }

S
Shuduo Sang 已提交
3001
    uint64_t  currentPrintTime = taosGetTimestampMs();
3002
    if (currentPrintTime - lastPrintTime > 30*1000) {
3003
      printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n",
3004
              pThreadInfo->threadID, pThreadInfo->start_table_from, i);
3005 3006 3007
      lastPrintTime = currentPrintTime;
    }
  }
3008

3009
  if (0 != len) {
3010 3011 3012
    if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer,
                NO_INSERT_TYPE, false)) {
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer);
3013
    }
3014
  }
3015

3016
  free(pThreadInfo->buffer);
3017 3018 3019
  return NULL;
}

3020
static int startMultiThreadCreateChildTable(
3021
        char* cols, int threads, uint64_t tableFrom, int64_t ntables,
3022
        char* db_name, SSuperTable* superTblInfo) {
3023

3024
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
3025
  threadInfo *infos = calloc(1, threads * sizeof(threadInfo));
3026 3027 3028 3029 3030 3031 3032 3033 3034 3035

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

S
Shuduo Sang 已提交
3036
  int64_t a = ntables / threads;
3037 3038 3039 3040 3041
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

S
Shuduo Sang 已提交
3042
  int64_t b = 0;
3043
  b = ntables % threads;
3044

3045 3046 3047
  for (int64_t i = 0; i < threads; i++) {
    threadInfo *pThreadInfo = infos + i;
    pThreadInfo->threadID = i;
3048
    tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN);
3049
    pThreadInfo->superTblInfo = superTblInfo;
3050
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
3051
    pThreadInfo->taos = taos_connect(
3052 3053 3054 3055 3056
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
3057
    if (pThreadInfo->taos == NULL) {
3058
      errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n",
3059
         __func__, __LINE__, taos_errstr(NULL));
3060
      free(pids);
3061
      free(infos);
3062 3063
      return -1;
    }
3064

3065
    pThreadInfo->start_table_from = tableFrom;
3066
    pThreadInfo->ntables = i<b?a+1:a;
3067 3068
    pThreadInfo->end_table_to = i < b ? tableFrom + a : tableFrom + a - 1;
    tableFrom = pThreadInfo->end_table_to + 1;
3069 3070 3071 3072
    pThreadInfo->use_metric = true;
    pThreadInfo->cols = cols;
    pThreadInfo->minDelay = UINT64_MAX;
    pthread_create(pids + i, NULL, createTable, pThreadInfo);
3073
  }
3074

3075 3076 3077 3078 3079
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
3080 3081
    threadInfo *pThreadInfo = infos + i;
    taos_close(pThreadInfo->taos);
3082 3083 3084
  }

  free(pids);
3085
  free(infos);
3086 3087

  return 0;
3088 3089
}

3090
static void createChildTables() {
3091 3092
    char tblColsBuf[MAX_SQL_SIZE];
    int len;
3093

3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133
    for (int i = 0; i < g_Dbs.dbCount; i++) {
        if (g_Dbs.use_metric) {
            if (g_Dbs.db[i].superTblCount > 0) {
                // with super table
                for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
                    if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
                            || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
                        continue;
                    }
                    verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
                          g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
                    uint64_t startFrom = 0;
                    g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;

                    verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n",
                            __func__, __LINE__, g_totalChildTables, startFrom);

                    startMultiThreadCreateChildTable(
                            g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
                            g_Dbs.threadCountByCreateTbl,
                            startFrom,
                            g_Dbs.db[i].superTbls[j].childTblCount,
                            g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
                }
            }
        } else {
            // normal table
            len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
            for (int j = 0; j < g_args.num_of_CPR; j++) {
                if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
                        || (strncasecmp(g_args.datatype[j],
                                "NCHAR", strlen("NCHAR")) == 0)) {
                    snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                            ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary);
                } else {
                    snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                            ", COL%d %s", j, g_args.datatype[j]);
                }
                len = strlen(tblColsBuf);
            }
3134

3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146
            snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");

            verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n",
                    __func__, __LINE__,
                    g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
            startMultiThreadCreateChildTable(
                    tblColsBuf,
                    g_Dbs.threadCountByCreateTbl,
                    0,
                    g_args.num_of_tables,
                    g_Dbs.db[i].dbName,
                    NULL);
3147
        }
3148
    }
3149 3150 3151 3152 3153
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3154
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
3155 3156 3157
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
3158

3159 3160
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
3161
    printf("Failed to open tags file: %s, reason:%s\n",
3162
            superTblInfo->tagsFile, strerror(errno));
3163 3164 3165 3166 3167 3168 3169
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
3170

3171 3172 3173 3174 3175 3176 3177 3178 3179
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

3180
  while((readLen = tgetline(&line, &n, fp)) != -1) {
3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
3193
      char *tmp = realloc(tagDataBuf,
3194
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
3195 3196 3197
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
3198
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
3199
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3222
static int readSampleFromCsvFileToMem(
3223
        SSuperTable* superTblInfo) {
3224 3225 3226 3227
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
3228

3229 3230
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
3231 3232 3233
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
              superTblInfo->sampleFile, strerror(errno));
      return -1;
3234
  }
3235

3236
  assert(superTblInfo->sampleDataBuf);
3237
  memset(superTblInfo->sampleDataBuf, 0,
3238
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
3239
  while(1) {
3240
    readLen = tgetline(&line, &n, fp);
3241 3242
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
3243
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
3244
                superTblInfo->sampleFile, strerror(errno));
3245
        fclose(fp);
3246 3247 3248 3249
        return -1;
      }
      continue;
    }
3250

3251 3252 3253 3254 3255 3256 3257 3258 3259
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
3260
      printf("sample row len[%d] overflow define schema len[%"PRIu64"], so discard this row\n",
3261
              (int32_t)readLen, superTblInfo->lenOfOneRow);
3262 3263 3264
      continue;
    }

3265
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
3266
          line, readLen);
3267 3268 3269 3270 3271 3272 3273
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

3274
  fclose(fp);
3275 3276 3277 3278
  tmfree(line);
  return 0;
}

3279
static bool getColumnAndTagTypeFromInsertJsonFile(
3280
        cJSON* stbInfo, SSuperTable* superTbls) {
3281
  bool  ret = false;
3282

3283
  // columns
3284 3285
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
3286
    printf("ERROR: failed to read json, columns not found\n");
3287 3288 3289 3290 3291 3292
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
3293

3294
  int columnSize = cJSON_GetArraySize(columns);
3295
  if ((columnSize + 1/* ts */) > MAX_COLUMN_COUNT) {
3296
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
3297
            __func__, __LINE__, MAX_COLUMN_COUNT);
3298 3299 3300 3301 3302 3303
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
3304

3305
  //superTbls->columnCount = columnSize;
3306 3307 3308 3309 3310 3311 3312
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
3313
      count = countObj->valueint;
3314
    } else if (countObj && countObj->type != cJSON_Number) {
3315
      errorPrint("%s() LN%d, failed to read json, column count not found\n",
3316
          __func__, __LINE__);
3317 3318 3319 3320 3321
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3322
    // column info
3323 3324
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
3325 3326 3327
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d: failed to read json, column type not found\n",
3328
          __func__, __LINE__);
3329 3330
      goto PARSE_OVER;
    }
3331
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3332
    tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1);
3333

3334 3335
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3336
      columnCase.dataLen = dataLen->valueint;
3337
    } else if (dataLen && dataLen->type != cJSON_Number) {
3338
      debugPrint("%s() LN%d: failed to read json, column len not found\n",
3339
          __func__, __LINE__);
3340 3341 3342 3343
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
3344

3345
    for (int n = 0; n < count; ++n) {
3346
      tstrncpy(superTbls->columns[index].dataType,
3347
              columnCase.dataType, strlen(columnCase.dataType) + 1);
3348
      superTbls->columns[index].dataLen = columnCase.dataLen;
3349 3350
      index++;
    }
3351
  }
3352

3353 3354
  if ((index + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n",
3355
            __func__, __LINE__, MAX_COLUMN_COUNT);
3356 3357 3358
    goto PARSE_OVER;
  }

3359
  superTbls->columnCount = index;
3360

3361 3362
  count = 1;
  index = 0;
3363
  // tags
3364 3365
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
3366
    errorPrint("%s() LN%d, failed to read json, tags not found\n",
3367
        __func__, __LINE__);
3368 3369
    goto PARSE_OVER;
  }
3370

3371 3372
  int tagSize = cJSON_GetArraySize(tags);
  if (tagSize > MAX_TAG_COUNT) {
3373
    errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n",
3374
        __func__, __LINE__, MAX_TAG_COUNT);
3375 3376
    goto PARSE_OVER;
  }
3377 3378

  //superTbls->tagCount = tagSize;
3379 3380 3381
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
3382

3383 3384 3385
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
3386
      count = countObj->valueint;
3387
    } else if (countObj && countObj->type != cJSON_Number) {
3388
      printf("ERROR: failed to read json, column count not found\n");
3389 3390 3391 3392 3393
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3394
    // column info
3395 3396
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
3397 3398 3399
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d, failed to read json, tag type not found\n",
3400
          __func__, __LINE__);
3401 3402
      goto PARSE_OVER;
    }
3403
    tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1);
3404

3405 3406
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3407
      columnCase.dataLen = dataLen->valueint;
3408
    } else if (dataLen && dataLen->type != cJSON_Number) {
3409
      errorPrint("%s() LN%d, failed to read json, column len not found\n",
3410
          __func__, __LINE__);
3411 3412 3413
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
3414 3415
    }

3416
    for (int n = 0; n < count; ++n) {
3417
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType,
3418
          strlen(columnCase.dataType) + 1);
3419
      superTbls->tags[index].dataLen = columnCase.dataLen;
3420 3421
      index++;
    }
3422
  }
3423 3424

  if (index > MAX_TAG_COUNT) {
3425
    errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n",
3426
        __func__, __LINE__, MAX_TAG_COUNT);
3427 3428 3429
    goto PARSE_OVER;
  }

3430 3431
  superTbls->tagCount = index;

3432 3433
  if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n",
3434
        __func__, __LINE__, MAX_COLUMN_COUNT);
3435 3436
    goto PARSE_OVER;
  }
3437 3438
  ret = true;

3439
PARSE_OVER:
3440 3441 3442 3443 3444 3445 3446 3447
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3448
    tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3449 3450 3451 3452
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3453
    tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE);
3454
  } else if (!host) {
3455
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3456
  } else {
3457
    printf("ERROR: failed to read json, host not found\n");
3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_Dbs.port = port->valueint;
  } else if (!port) {
    g_Dbs.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3470
    tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE);
3471
  } else if (!user) {
3472
    tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE);
3473 3474 3475 3476
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3477
    tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE);
3478
  } else if (!password) {
3479
    tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE);
3480 3481 3482 3483
  }

  cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
  if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
3484
    tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
3485
  } else if (!resultfile) {
3486
    tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
3487 3488 3489 3490 3491 3492 3493 3494
  }

  cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
  if (threads && threads->type == cJSON_Number) {
    g_Dbs.threadCount = threads->valueint;
  } else if (!threads) {
    g_Dbs.threadCount = 1;
  } else {
3495
    printf("ERROR: failed to read json, threads not found\n");
3496
    goto PARSE_OVER;
3497 3498
  }

3499 3500 3501 3502
  cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
  if (threads2 && threads2->type == cJSON_Number) {
    g_Dbs.threadCountByCreateTbl = threads2->valueint;
  } else if (!threads2) {
sangshuduo's avatar
sangshuduo 已提交
3503
    g_Dbs.threadCountByCreateTbl = 1;
3504
  } else {
3505
    errorPrint("%s() LN%d, failed to read json, threads2 not found\n",
3506
            __func__, __LINE__);
3507
    goto PARSE_OVER;
3508
  }
3509

3510 3511
  cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
  if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
3512 3513 3514 3515 3516
    if (gInsertInterval->valueint <0) {
      errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n",
            __func__, __LINE__);
      goto PARSE_OVER;
    }
3517 3518 3519 3520
    g_args.insert_interval = gInsertInterval->valueint;
  } else if (!gInsertInterval) {
    g_args.insert_interval = 0;
  } else {
3521
    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
3522
        __func__, __LINE__);
3523 3524
    goto PARSE_OVER;
  }
3525

3526 3527
  cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows");
  if (interlaceRows && interlaceRows->type == cJSON_Number) {
3528 3529 3530 3531 3532 3533
    if (interlaceRows->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;

    }
3534 3535
    g_args.interlace_rows = interlaceRows->valueint;
  } else if (!interlaceRows) {
3536
    g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3537
  } else {
3538
    errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
3539
        __func__, __LINE__);
3540
    goto PARSE_OVER;
3541
  }
3542

3543 3544
  cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
  if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
3545 3546 3547 3548 3549
    if (maxSqlLen->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
    }
3550 3551
    g_args.max_sql_len = maxSqlLen->valueint;
  } else if (!maxSqlLen) {
sangshuduo's avatar
sangshuduo 已提交
3552
    g_args.max_sql_len = (1024*1024);
3553
  } else {
3554
    errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
3555
        __func__, __LINE__);
3556 3557 3558 3559 3560
    goto PARSE_OVER;
  }

  cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
  if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
3561
    if (numRecPerReq->valueint <= 0) {
3562 3563 3564
      errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
3565
    } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) {
3566 3567 3568 3569 3570
      printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n",
              numRecPerReq->valueint, MAX_RECORDS_PER_REQ);
      printf("        number of records per request value will be set to %d\n\n",
              MAX_RECORDS_PER_REQ);
      prompt();
3571
      numRecPerReq->valueint = MAX_RECORDS_PER_REQ;
3572
    }
3573 3574
    g_args.num_of_RPR = numRecPerReq->valueint;
  } else if (!numRecPerReq) {
3575
    g_args.num_of_RPR = MAX_RECORDS_PER_REQ;
3576
  } else {
3577
    errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n",
3578
        __func__, __LINE__);
3579 3580 3581
    goto PARSE_OVER;
  }

3582
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3583
  if (answerPrompt
3584 3585
          && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3586 3587 3588 3589 3590 3591 3592 3593
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
    } else {
      g_args.answer_yes = false;
    }
  } else if (!answerPrompt) {
3594
    g_args.answer_yes = true;   // default is no, mean answer_yes.
3595
  } else {
3596
    errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n");
3597
    goto PARSE_OVER;
3598
  }
3599

3600 3601
  // rows per table need be less than insert batch
  if (g_args.interlace_rows > g_args.num_of_RPR) {
3602
      printf("NOTICE: interlace rows value %u > num_of_records_per_req %u\n\n",
3603
              g_args.interlace_rows, g_args.num_of_RPR);
3604
      printf("        interlace rows value will be set to num_of_records_per_req %u\n\n",
3605 3606 3607 3608 3609
              g_args.num_of_RPR);
      prompt();
      g_args.interlace_rows = g_args.num_of_RPR;
  }

3610 3611
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (!dbs || dbs->type != cJSON_Array) {
3612
    printf("ERROR: failed to read json, databases not found\n");
3613 3614 3615 3616 3617
    goto PARSE_OVER;
  }

  int dbSize = cJSON_GetArraySize(dbs);
  if (dbSize > MAX_DB_COUNT) {
3618
    errorPrint(
3619 3620
            "ERROR: failed to read json, databases size overflow, max database is %d\n",
            MAX_DB_COUNT);
3621 3622 3623 3624 3625 3626 3627 3628
    goto PARSE_OVER;
  }

  g_Dbs.dbCount = dbSize;
  for (int i = 0; i < dbSize; ++i) {
    cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
    if (dbinfos == NULL) continue;

3629
    // dbinfo
3630 3631
    cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
    if (!dbinfo || dbinfo->type != cJSON_Object) {
3632
      printf("ERROR: failed to read json, dbinfo not found\n");
3633 3634
      goto PARSE_OVER;
    }
3635

3636 3637
    cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
    if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
3638
      printf("ERROR: failed to read json, db name not found\n");
3639 3640
      goto PARSE_OVER;
    }
3641
    tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, TSDB_DB_NAME_LEN);
3642 3643 3644

    cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
    if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
3645 3646
      if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) {
        g_Dbs.db[i].drop = true;
3647
      } else {
3648
        g_Dbs.db[i].drop = false;
3649
      }
3650
    } else if (!drop) {
3651
      g_Dbs.db[i].drop = g_args.drop_database;
3652
    } else {
3653
      errorPrint("%s() LN%d, failed to read json, drop input mistake\n",
3654
              __func__, __LINE__);
3655 3656 3657 3658
      goto PARSE_OVER;
    }

    cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
3659
    if (precision && precision->type == cJSON_String
3660
            && precision->valuestring != NULL) {
3661
      tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
3662
              8);
3663
    } else if (!precision) {
3664
      memset(g_Dbs.db[i].dbCfg.precision, 0, 8);
3665
    } else {
3666
      printf("ERROR: failed to read json, precision not found\n");
3667 3668 3669 3670 3671 3672 3673 3674 3675
      goto PARSE_OVER;
    }

    cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
    if (update && update->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.update = update->valueint;
    } else if (!update) {
      g_Dbs.db[i].dbCfg.update = -1;
    } else {
3676
      printf("ERROR: failed to read json, update not found\n");
3677 3678 3679 3680 3681 3682 3683 3684 3685
      goto PARSE_OVER;
    }

    cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
    if (replica && replica->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.replica = replica->valueint;
    } else if (!replica) {
      g_Dbs.db[i].dbCfg.replica = -1;
    } else {
3686
      printf("ERROR: failed to read json, replica not found\n");
3687 3688 3689 3690 3691 3692 3693 3694 3695
      goto PARSE_OVER;
    }

    cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
    if (keep && keep->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.keep = keep->valueint;
    } else if (!keep) {
      g_Dbs.db[i].dbCfg.keep = -1;
    } else {
3696 3697
     printf("ERROR: failed to read json, keep not found\n");
     goto PARSE_OVER;
3698
    }
3699

3700 3701 3702 3703 3704 3705
    cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
    if (days && days->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.days = days->valueint;
    } else if (!days) {
      g_Dbs.db[i].dbCfg.days = -1;
    } else {
3706 3707
     printf("ERROR: failed to read json, days not found\n");
     goto PARSE_OVER;
3708
    }
3709

3710 3711 3712 3713 3714 3715
    cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
    if (cache && cache->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cache = cache->valueint;
    } else if (!cache) {
      g_Dbs.db[i].dbCfg.cache = -1;
    } else {
3716 3717
     printf("ERROR: failed to read json, cache not found\n");
     goto PARSE_OVER;
3718
    }
3719

3720 3721 3722 3723 3724 3725
    cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
    if (blocks && blocks->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
    } else if (!blocks) {
      g_Dbs.db[i].dbCfg.blocks = -1;
    } else {
3726 3727
     printf("ERROR: failed to read json, block not found\n");
     goto PARSE_OVER;
3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743
    }

    //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
    //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
    //} else if (!maxtablesPerVnode) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
    //} else {
    // printf("failed to read json, maxtablesPerVnode not found");
    // goto PARSE_OVER;
    //}

    cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
    if (minRows && minRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
    } else if (!minRows) {
3744
      g_Dbs.db[i].dbCfg.minRows = 0;    // 0 means default
3745
    } else {
3746 3747
     printf("ERROR: failed to read json, minRows not found\n");
     goto PARSE_OVER;
3748 3749 3750 3751 3752 3753
    }

    cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
    if (maxRows && maxRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
    } else if (!maxRows) {
3754
      g_Dbs.db[i].dbCfg.maxRows = 0;    // 0 means default
3755
    } else {
3756 3757
     printf("ERROR: failed to read json, maxRows not found\n");
     goto PARSE_OVER;
3758 3759 3760 3761 3762 3763 3764 3765
    }

    cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
    if (comp && comp->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.comp = comp->valueint;
    } else if (!comp) {
      g_Dbs.db[i].dbCfg.comp = -1;
    } else {
3766 3767
     printf("ERROR: failed to read json, comp not found\n");
     goto PARSE_OVER;
3768 3769 3770 3771 3772 3773 3774 3775
    }

    cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
    if (walLevel && walLevel->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
    } else if (!walLevel) {
      g_Dbs.db[i].dbCfg.walLevel = -1;
    } else {
3776 3777
     printf("ERROR: failed to read json, walLevel not found\n");
     goto PARSE_OVER;
3778 3779
    }

3780 3781 3782 3783 3784 3785
    cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
    if (cacheLast && cacheLast->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
    } else if (!cacheLast) {
      g_Dbs.db[i].dbCfg.cacheLast = -1;
    } else {
3786 3787
     printf("ERROR: failed to read json, cacheLast not found\n");
     goto PARSE_OVER;
3788 3789
    }

3790 3791 3792 3793
    cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
    if (quorum && quorum->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
    } else if (!quorum) {
3794
      g_Dbs.db[i].dbCfg.quorum = 1;
3795
    } else {
3796 3797
     printf("failed to read json, quorum input mistake");
     goto PARSE_OVER;
3798 3799 3800 3801 3802 3803 3804 3805
    }

    cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
    if (fsync && fsync->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
    } else if (!fsync) {
      g_Dbs.db[i].dbCfg.fsync = -1;
    } else {
3806
      errorPrint("%s() LN%d, failed to read json, fsync input mistake\n",
3807
              __func__, __LINE__);
3808
      goto PARSE_OVER;
3809
    }
3810

3811
    // super_talbes
3812 3813
    cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
    if (!stables || stables->type != cJSON_Array) {
3814
      errorPrint("%s() LN%d, failed to read json, super_tables not found\n",
3815
              __func__, __LINE__);
3816
      goto PARSE_OVER;
3817 3818
    }

3819 3820
    int stbSize = cJSON_GetArraySize(stables);
    if (stbSize > MAX_SUPER_TABLE_COUNT) {
3821
      errorPrint(
3822 3823
              "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n",
              __func__, __LINE__, MAX_SUPER_TABLE_COUNT);
3824 3825 3826 3827 3828 3829 3830
      goto PARSE_OVER;
    }

    g_Dbs.db[i].superTblCount = stbSize;
    for (int j = 0; j < stbSize; ++j) {
      cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
      if (stbInfo == NULL) continue;
3831 3832

      // dbinfo
3833
      cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
3834 3835
      if (!stbName || stbName->type != cJSON_String
              || stbName->valuestring == NULL) {
3836
        errorPrint("%s() LN%d, failed to read json, stb name not found\n",
3837
                __func__, __LINE__);
3838 3839
        goto PARSE_OVER;
      }
3840
      tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring,
3841
              TSDB_TABLE_NAME_LEN);
3842

3843 3844
      cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
      if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
3845
        printf("ERROR: failed to read json, childtable_prefix not found\n");
3846 3847
        goto PARSE_OVER;
      }
3848
      tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring,
3849
              TSDB_TABLE_NAME_LEN - 20);
3850

3851
      cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table");
3852
      if (autoCreateTbl
3853 3854
              && autoCreateTbl->type == cJSON_String
              && autoCreateTbl->valuestring != NULL) {
3855 3856 3857 3858 3859 3860 3861 3862
          if ((0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3))
                  && (TBL_ALREADY_EXISTS != g_Dbs.db[i].superTbls[j].childTblExists)) {
              g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
          } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
              g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
          } else {
              g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
          }
3863
      } else if (!autoCreateTbl) {
3864
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
3865
      } else {
3866 3867
          printf("ERROR: failed to read json, auto_create_table not found\n");
          goto PARSE_OVER;
3868
      }
3869

3870 3871 3872 3873
      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
      } else if (!batchCreateTbl) {
3874
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
3875
      } else {
3876
        printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
3877
        goto PARSE_OVER;
3878
      }
3879 3880

      cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
3881
      if (childTblExists
3882 3883
              && childTblExists->type == cJSON_String
              && childTblExists->valuestring != NULL) {
3884 3885
        if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3))
            && (g_Dbs.db[i].drop == false)) {
3886
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
3887
        } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2)
3888
              || (g_Dbs.db[i].drop == true))) {
3889 3890 3891 3892 3893 3894 3895
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        } else {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        }
      } else if (!childTblExists) {
        g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
      } else {
3896
        errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n",
3897
                __func__, __LINE__);
3898 3899
        goto PARSE_OVER;
      }
3900

3901 3902 3903 3904
      if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      }

3905 3906
      cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
      if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
3907
        errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n",
3908
                __func__, __LINE__);
3909 3910 3911 3912 3913
        goto PARSE_OVER;
      }
      g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

      cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
3914
      if (dataSource && dataSource->type == cJSON_String
3915
              && dataSource->valuestring != NULL) {
3916
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
3917
                dataSource->valuestring, TSDB_DB_NAME_LEN);
3918
      } else if (!dataSource) {
3919
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", TSDB_DB_NAME_LEN);
3920
      } else {
3921
        errorPrint("%s() LN%d, failed to read json, data_source not found\n",
3922
            __func__, __LINE__);
3923 3924 3925
        goto PARSE_OVER;
      }

3926 3927 3928 3929 3930 3931 3932
      cJSON *stbIface = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt
      if (stbIface && stbIface->type == cJSON_String
              && stbIface->valuestring != NULL) {
        if (0 == strcasecmp(stbIface->valuestring, "taosc")) {
            g_Dbs.db[i].superTbls[j].iface= TAOSC_IFACE;
        } else if (0 == strcasecmp(stbIface->valuestring, "rest")) {
            g_Dbs.db[i].superTbls[j].iface= REST_IFACE;
3933
#if STMT_IFACE_ENABLED == 1
3934 3935
        } else if (0 == strcasecmp(stbIface->valuestring, "stmt")) {
            g_Dbs.db[i].superTbls[j].iface= STMT_IFACE;
3936
#endif
3937 3938
        } else {
            errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n",
3939
                    __func__, __LINE__, stbIface->valuestring);
3940 3941
            goto PARSE_OVER;
        }
3942 3943
      } else if (!stbIface) {
        g_Dbs.db[i].superTbls[j].iface = TAOSC_IFACE;
3944
      } else {
3945
        errorPrint("%s", "failed to read json, insert_mode not found\n");
3946 3947 3948
        goto PARSE_OVER;
      }

3949
      cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
3950 3951
      if ((childTbl_limit) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3952
        if (childTbl_limit->type != cJSON_Number) {
3953 3954
            printf("ERROR: failed to read json, childtable_limit\n");
            goto PARSE_OVER;
3955 3956 3957
        }
        g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
      } else {
3958
        g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid.
3959 3960 3961
      }

      cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
3962 3963
      if ((childTbl_offset) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3964 3965
        if ((childTbl_offset->type != cJSON_Number)
                || (0 > childTbl_offset->valueint)) {
3966 3967
            printf("ERROR: failed to read json, childtable_offset\n");
            goto PARSE_OVER;
3968 3969 3970 3971 3972 3973
        }
        g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblOffset = 0;
      }

3974 3975
      cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
      if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
3976
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
3977
                ts->valuestring, TSDB_DB_NAME_LEN);
3978
      } else if (!ts) {
3979
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
3980
                "now", TSDB_DB_NAME_LEN);
3981
      } else {
3982
        printf("ERROR: failed to read json, start_timestamp not found\n");
3983 3984
        goto PARSE_OVER;
      }
3985

3986 3987 3988 3989
      cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
      if (timestampStep && timestampStep->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
      } else if (!timestampStep) {
3990
        g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
3991
      } else {
3992
        printf("ERROR: failed to read json, timestamp_step not found\n");
3993 3994 3995 3996
        goto PARSE_OVER;
      }

      cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
3997
      if (sampleFormat && sampleFormat->type
3998
              == cJSON_String && sampleFormat->valuestring != NULL) {
3999
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
4000
                sampleFormat->valuestring, TSDB_DB_NAME_LEN);
4001
      } else if (!sampleFormat) {
4002
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", TSDB_DB_NAME_LEN);
4003
      } else {
4004
        printf("ERROR: failed to read json, sample_format not found\n");
4005
        goto PARSE_OVER;
4006
      }
4007

4008
      cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
4009 4010
      if (sampleFile && sampleFile->type == cJSON_String
          && sampleFile->valuestring != NULL) {
4011
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
4012
                sampleFile->valuestring, MAX_FILE_NAME_LEN);
4013 4014 4015
      } else if (!sampleFile) {
        memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
      } else {
4016
        printf("ERROR: failed to read json, sample_file not found\n");
4017
        goto PARSE_OVER;
4018
      }
4019

4020
      cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
4021 4022
      if ((tagsFile && tagsFile->type == cJSON_String)
              && (tagsFile->valuestring != NULL)) {
4023
        tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
4024
                tagsFile->valuestring, MAX_FILE_NAME_LEN);
4025 4026 4027 4028 4029 4030 4031 4032 4033
        if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
          g_Dbs.db[i].superTbls[j].tagSource = 0;
        } else {
          g_Dbs.db[i].superTbls[j].tagSource = 1;
        }
      } else if (!tagsFile) {
        memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
        g_Dbs.db[i].superTbls[j].tagSource = 0;
      } else {
4034
        printf("ERROR: failed to read json, tags_file not found\n");
4035 4036
        goto PARSE_OVER;
      }
4037

sangshuduo's avatar
sangshuduo 已提交
4038 4039 4040
      cJSON* stbMaxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
      if (stbMaxSqlLen && stbMaxSqlLen->type == cJSON_Number) {
        int32_t len = stbMaxSqlLen->valueint;
4041 4042
        if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
          len = TSDB_MAX_ALLOWED_SQL_LEN;
4043 4044
        } else if (len < 5) {
          len = 5;
4045
        }
4046 4047
        g_Dbs.db[i].superTbls[j].maxSqlLen = len;
      } else if (!maxSqlLen) {
4048
        g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len;
4049
      } else {
sangshuduo's avatar
sangshuduo 已提交
4050
        errorPrint("%s() LN%d, failed to read json, stbMaxSqlLen input mistake\n",
4051
            __func__, __LINE__);
4052
        goto PARSE_OVER;
4053
      }
4054
/*
4055 4056 4057 4058 4059
      cJSON *multiThreadWriteOneTbl =
          cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
      if (multiThreadWriteOneTbl
              && multiThreadWriteOneTbl->type == cJSON_String
              && multiThreadWriteOneTbl->valuestring != NULL) {
4060 4061 4062 4063
        if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
        } else {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
4064
        }
4065 4066 4067
      } else if (!multiThreadWriteOneTbl) {
        g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
      } else {
4068
        printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
4069 4070
        goto PARSE_OVER;
      }
4071
*/
sangshuduo's avatar
sangshuduo 已提交
4072 4073 4074
      cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows");
      if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) {
        if (stbInterlaceRows->valueint < 0) {
4075 4076 4077 4078
          errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n",
            __func__, __LINE__);
          goto PARSE_OVER;
        }
sangshuduo's avatar
sangshuduo 已提交
4079
        g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint;
4080 4081
        // rows per table need be less than insert batch
        if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) {
4082
          printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %u > num_of_records_per_req %u\n\n",
4083 4084
                  i, j, g_Dbs.db[i].superTbls[j].interlaceRows,
                  g_args.num_of_RPR);
4085
          printf("        interlace rows value will be set to num_of_records_per_req %u\n\n",
4086
                  g_args.num_of_RPR);
4087
          prompt();
4088 4089
          g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR;
        }
sangshuduo's avatar
sangshuduo 已提交
4090
      } else if (!stbInterlaceRows) {
4091
        g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
4092
      } else {
4093
        errorPrint(
4094 4095
                "%s() LN%d, failed to read json, interlace rows input mistake\n",
                __func__, __LINE__);
4096
        goto PARSE_OVER;
4097
      }
4098 4099 4100

      cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
      if (disorderRatio && disorderRatio->type == cJSON_Number) {
4101 4102 4103 4104 4105 4106
        if (disorderRatio->valueint > 50)
          disorderRatio->valueint = 50;

        if (disorderRatio->valueint < 0)
          disorderRatio->valueint = 0;

4107 4108 4109 4110
        g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
      } else if (!disorderRatio) {
        g_Dbs.db[i].superTbls[j].disorderRatio = 0;
      } else {
4111
        printf("ERROR: failed to read json, disorderRatio not found\n");
4112
        goto PARSE_OVER;
4113
      }
4114 4115 4116 4117 4118 4119 4120

      cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
      if (disorderRange && disorderRange->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
      } else if (!disorderRange) {
        g_Dbs.db[i].superTbls[j].disorderRange = 1000;
      } else {
4121
        printf("ERROR: failed to read json, disorderRange not found\n");
4122 4123
        goto PARSE_OVER;
      }
4124

4125 4126
      cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
      if (insertRows && insertRows->type == cJSON_Number) {
4127 4128 4129 4130 4131
        if (insertRows->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4132 4133 4134 4135
        g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
      } else if (!insertRows) {
        g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
      } else {
4136
        errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
4137
                __func__, __LINE__);
4138 4139 4140 4141 4142 4143
        goto PARSE_OVER;
      }

      cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
      if (insertInterval && insertInterval->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
4144 4145 4146 4147 4148
        if (insertInterval->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4149
      } else if (!insertInterval) {
4150
        verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n",
4151
                __func__, __LINE__, g_args.insert_interval);
4152 4153
        g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
      } else {
4154
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
4155
                __func__, __LINE__);
4156 4157 4158
        goto PARSE_OVER;
      }

4159
      int retVal = getColumnAndTagTypeFromInsertJsonFile(
4160
              stbInfo, &g_Dbs.db[i].superTbls[j]);
4161 4162
      if (false == retVal) {
        goto PARSE_OVER;
4163 4164
      }
    }
4165 4166 4167 4168
  }

  ret = true;

4169
PARSE_OVER:
4170 4171 4172 4173 4174 4175 4176 4177
  return ret;
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
4178
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
4179 4180 4181 4182
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
4183
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE);
4184
  } else if (!host) {
4185
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
4186
  } else {
4187
    printf("ERROR: failed to read json, host not found\n");
4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
4200
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE);
4201
  } else if (!user) {
4202
    tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ;
4203 4204 4205 4206
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
4207
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE);
4208
  } else if (!password) {
4209
    tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);;
4210
  }
H
hzcheng 已提交
4211

4212
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
4213
  if (answerPrompt && answerPrompt->type == cJSON_String
4214
          && answerPrompt->valuestring != NULL) {
4215 4216 4217 4218
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
4219
    } else {
4220
      g_args.answer_yes = false;
H
hzcheng 已提交
4221
    }
4222 4223 4224
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
4225
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
4226
    goto PARSE_OVER;
4227
  }
4228

4229 4230
  cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
  if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
4231
    if (gQueryTimes->valueint <= 0) {
4232 4233
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4234 4235
      goto PARSE_OVER;
    }
4236 4237 4238 4239
    g_args.query_times = gQueryTimes->valueint;
  } else if (!gQueryTimes) {
    g_args.query_times = 1;
  } else {
4240
    errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4241
        __func__, __LINE__);
4242 4243 4244
    goto PARSE_OVER;
  }

4245 4246
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
4247
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, TSDB_DB_NAME_LEN);
4248
  } else if (!dbs) {
4249
    printf("ERROR: failed to read json, databases not found\n");
4250
    goto PARSE_OVER;
H
hzcheng 已提交
4251
  }
4252 4253 4254

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
4255
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
4256
  } else if (!queryMode) {
4257
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
4258
  } else {
4259
    printf("ERROR: failed to read json, query_mode not found\n");
4260
    goto PARSE_OVER;
H
hzcheng 已提交
4261
  }
4262

4263
  // specified_table_query
4264 4265
  cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!specifiedQuery) {
4266
    g_queryInfo.specifiedQueryInfo.concurrent = 1;
4267 4268
    g_queryInfo.specifiedQueryInfo.sqlCount = 0;
  } else if (specifiedQuery->type != cJSON_Object) {
4269
    printf("ERROR: failed to read json, super_table_query not found\n");
4270
    goto PARSE_OVER;
4271
  } else {
4272 4273 4274 4275 4276
    cJSON* queryInterval = cJSON_GetObjectItem(specifiedQuery, "query_interval");
    if (queryInterval && queryInterval->type == cJSON_Number) {
      g_queryInfo.specifiedQueryInfo.queryInterval = queryInterval->valueint;
    } else if (!queryInterval) {
      g_queryInfo.specifiedQueryInfo.queryInterval = 0;
4277 4278
    }

4279
    cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery,
4280
        "query_times");
4281
    if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
4282
      if (specifiedQueryTimes->valueint <= 0) {
4283 4284
        errorPrint(
                "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4285
          __func__, __LINE__, specifiedQueryTimes->valueint);
4286 4287 4288
        goto PARSE_OVER;

      }
4289 4290 4291 4292
      g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint;
    } else if (!specifiedQueryTimes) {
      g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times;
    } else {
4293
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4294
          __func__, __LINE__);
4295
      goto PARSE_OVER;
4296
    }
4297

4298
    cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent");
4299
    if (concurrent && concurrent->type == cJSON_Number) {
4300
      if (concurrent->valueint <= 0) {
4301
        errorPrint(
4302
                "%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n",
4303 4304
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount,
4305
              g_queryInfo.specifiedQueryInfo.concurrent);
4306 4307
        goto PARSE_OVER;
      }
4308
      g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint;
4309
    } else if (!concurrent) {
4310
      g_queryInfo.specifiedQueryInfo.concurrent = 1;
4311
    }
4312

4313 4314 4315 4316 4317 4318 4319
    cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode");
    if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String
        && specifiedAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE;
4320
      } else {
4321
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4322
            __func__, __LINE__);
4323 4324 4325
        goto PARSE_OVER;
      }
    } else {
4326
      g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
4327
    }
4328

4329
    cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval");
4330
    if (interval && interval->type == cJSON_Number) {
4331
      g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint;
4332
    } else if (!interval) {
4333 4334
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4335
      g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000;
4336
    }
4337

4338
    cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart");
4339
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
4340
      if (0 == strcmp("yes", restart->valuestring)) {
4341
        g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4342
      } else if (0 == strcmp("no", restart->valuestring)) {
4343
        g_queryInfo.specifiedQueryInfo.subscribeRestart = false;
4344
      } else {
4345
        printf("ERROR: failed to read json, subscribe restart error\n");
4346 4347 4348
        goto PARSE_OVER;
      }
    } else {
4349
      g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4350
    }
4351

4352
    cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress");
4353
    if (keepProgress
4354 4355
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
4356
      if (0 == strcmp("yes", keepProgress->valuestring)) {
4357
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1;
4358
      } else if (0 == strcmp("no", keepProgress->valuestring)) {
4359
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4360
      } else {
4361
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4362 4363 4364
        goto PARSE_OVER;
      }
    } else {
4365
      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4366 4367
    }

4368
    // sqls
4369 4370
    cJSON* specifiedSqls = cJSON_GetObjectItem(specifiedQuery, "sqls");
    if (!specifiedSqls) {
4371
      g_queryInfo.specifiedQueryInfo.sqlCount = 0;
4372
    } else if (specifiedSqls->type != cJSON_Array) {
4373
      errorPrint("%s() LN%d, failed to read json, super sqls not found\n",
4374
          __func__, __LINE__);
4375
      goto PARSE_OVER;
4376
    } else {
4377 4378 4379 4380 4381 4382 4383 4384
      int superSqlSize = cJSON_GetArraySize(specifiedSqls);
      if (superSqlSize * g_queryInfo.specifiedQueryInfo.concurrent
              > MAX_QUERY_SQL_COUNT) {
        errorPrint("%s() LN%d, failed to read json, query sql(%d) * concurrent(%d) overflow, max is %d\n",
           __func__, __LINE__,
           superSqlSize,
           g_queryInfo.specifiedQueryInfo.concurrent,
           MAX_QUERY_SQL_COUNT);
4385 4386
        goto PARSE_OVER;
      }
4387

4388
      g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize;
4389
      for (int j = 0; j < superSqlSize; ++j) {
4390
        cJSON* sql = cJSON_GetArrayItem(specifiedSqls, j);
4391
        if (sql == NULL) continue;
4392

4393 4394
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
4395
          printf("ERROR: failed to read json, sql not found\n");
4396 4397
          goto PARSE_OVER;
        }
4398 4399
        tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j],
                sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
H
hzcheng 已提交
4400

4401 4402
        // default value is -1, which mean infinite loop
        g_queryInfo.specifiedQueryInfo.endAfterConsume[j] = -1;
4403 4404 4405 4406 4407 4408 4409
        cJSON* endAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "endAfterConsume");
        if (endAfterConsume
                && endAfterConsume->type == cJSON_Number) {
            g_queryInfo.specifiedQueryInfo.endAfterConsume[j]
                = endAfterConsume->valueint;
        }
4410 4411
        if (g_queryInfo.specifiedQueryInfo.endAfterConsume[j] < -1)
            g_queryInfo.specifiedQueryInfo.endAfterConsume[j] = -1;
4412

4413
        g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = -1;
4414 4415
        cJSON* resubAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume");
4416 4417 4418
        if ((resubAfterConsume)
                && (resubAfterConsume->type == cJSON_Number)
                && (resubAfterConsume->valueint >= 0)) {
4419 4420 4421 4422
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j]
                = resubAfterConsume->valueint;
        }

4423 4424 4425
        if (g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] < -1)
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = -1;

4426
        cJSON *result = cJSON_GetObjectItem(sql, "result");
4427 4428 4429 4430
        if ((NULL != result) && (result->type == cJSON_String)
                && (result->valuestring != NULL)) {
          tstrncpy(g_queryInfo.specifiedQueryInfo.result[j],
                  result->valuestring, MAX_FILE_NAME_LEN);
4431
        } else if (NULL == result) {
4432 4433
          memset(g_queryInfo.specifiedQueryInfo.result[j],
                  0, MAX_FILE_NAME_LEN);
4434
        } else {
4435
          printf("ERROR: failed to read json, super query result file not found\n");
4436
          goto PARSE_OVER;
4437
        }
4438 4439 4440
      }
    }
  }
4441

4442
  // super_table_query
4443 4444
  cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!superQuery) {
4445
    g_queryInfo.superQueryInfo.threadCnt = 1;
4446 4447
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
4448
    printf("ERROR: failed to read json, sub_table_query not found\n");
4449 4450
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
4451
  } else {
4452
    cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval");
4453
    if (subrate && subrate->type == cJSON_Number) {
4454
      g_queryInfo.superQueryInfo.queryInterval = subrate->valueint;
4455
    } else if (!subrate) {
4456
      g_queryInfo.superQueryInfo.queryInterval = 0;
4457 4458 4459 4460
    }

    cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
    if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
4461 4462
      if (superQueryTimes->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4463
          __func__, __LINE__, superQueryTimes->valueint);
4464 4465
        goto PARSE_OVER;
      }
4466 4467 4468 4469
      g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
    } else if (!superQueryTimes) {
      g_queryInfo.superQueryInfo.queryTimes = g_args.query_times;
    } else {
4470
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4471
          __func__, __LINE__);
4472
      goto PARSE_OVER;
4473
    }
4474

4475
    cJSON* threads = cJSON_GetObjectItem(superQuery, "threads");
4476
    if (threads && threads->type == cJSON_Number) {
4477 4478 4479 4480 4481 4482
      if (threads->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, threads input mistake\n",
          __func__, __LINE__);
        goto PARSE_OVER;

      }
4483
      g_queryInfo.superQueryInfo.threadCnt = threads->valueint;
4484
    } else if (!threads) {
4485
      g_queryInfo.superQueryInfo.threadCnt = 1;
4486
    }
4487

4488
    //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count");
4489
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
4490
    //  g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint;
4491
    //} else if (!subTblCnt) {
4492
    //  g_queryInfo.superQueryInfo.childTblCount = 0;
4493
    //}
4494

4495
    cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname");
4496 4497 4498
    if (stblname && stblname->type == cJSON_String
        && stblname->valuestring != NULL) {
      tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring,
4499
          TSDB_TABLE_NAME_LEN);
4500
    } else {
4501
      errorPrint("%s() LN%d, failed to read json, super table name input error\n",
4502
          __func__, __LINE__);
4503 4504
      goto PARSE_OVER;
    }
4505

4506 4507 4508 4509 4510 4511 4512
    cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode");
    if (superAsyncMode && superAsyncMode->type == cJSON_String
        && superAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE;
4513
      } else {
4514
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4515
            __func__, __LINE__);
4516 4517 4518
        goto PARSE_OVER;
      }
    } else {
4519
      g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
4520
    }
4521

4522 4523 4524 4525 4526 4527 4528 4529 4530
    cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval");
    if (superInterval && superInterval->type == cJSON_Number) {
      if (superInterval->valueint < 0) {
        errorPrint("%s() LN%d, failed to read json, interval input mistake\n",
            __func__, __LINE__);
        goto PARSE_OVER;
      }
      g_queryInfo.superQueryInfo.subscribeInterval = superInterval->valueint;
    } else if (!superInterval) {
4531 4532
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4533
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
4534
    }
4535

4536
    cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart");
4537 4538
    if (subrestart && subrestart->type == cJSON_String
        && subrestart->valuestring != NULL) {
4539
      if (0 == strcmp("yes", subrestart->valuestring)) {
4540
        g_queryInfo.superQueryInfo.subscribeRestart = true;
4541
      } else if (0 == strcmp("no", subrestart->valuestring)) {
4542
        g_queryInfo.superQueryInfo.subscribeRestart = false;
4543
      } else {
4544
        printf("ERROR: failed to read json, subscribe restart error\n");
4545 4546 4547
        goto PARSE_OVER;
      }
    } else {
4548
      g_queryInfo.superQueryInfo.subscribeRestart = true;
4549
    }
4550

4551 4552 4553 4554 4555
    cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
    if (superkeepProgress &&
            superkeepProgress->type == cJSON_String
            && superkeepProgress->valuestring != NULL) {
      if (0 == strcmp("yes", superkeepProgress->valuestring)) {
4556
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
4557
      } else if (0 == strcmp("no", superkeepProgress->valuestring)) {
4558
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4559
      } else {
4560
        printf("ERROR: failed to read json, subscribe super table keepProgress error\n");
4561 4562 4563
        goto PARSE_OVER;
      }
    } else {
4564
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4565
    }
4566

4567 4568
    // default value is -1, which mean do not resub
    g_queryInfo.superQueryInfo.endAfterConsume = -1;
4569 4570 4571 4572 4573 4574 4575
    cJSON* superEndAfterConsume =
            cJSON_GetObjectItem(superQuery, "endAfterConsume");
    if (superEndAfterConsume
            && superEndAfterConsume->type == cJSON_Number) {
        g_queryInfo.superQueryInfo.endAfterConsume =
            superEndAfterConsume->valueint;
    }
4576 4577
    if (g_queryInfo.superQueryInfo.endAfterConsume < -1)
        g_queryInfo.superQueryInfo.endAfterConsume = -1;
4578

4579 4580
    // default value is -1, which mean do not resub
    g_queryInfo.superQueryInfo.resubAfterConsume = -1;
4581
    cJSON* superResubAfterConsume =
4582 4583 4584 4585 4586
            cJSON_GetObjectItem(superQuery, "resubAfterConsume");
    if ((superResubAfterConsume)
            && (superResubAfterConsume->type == cJSON_Number)
            && (superResubAfterConsume->valueint >= 0)) {
        g_queryInfo.superQueryInfo.resubAfterConsume =
4587 4588
            superResubAfterConsume->valueint;
    }
4589 4590
    if (g_queryInfo.superQueryInfo.resubAfterConsume < -1)
        g_queryInfo.superQueryInfo.resubAfterConsume = -1;
4591

4592 4593 4594
    // supert table sqls
    cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls");
    if (!superSqls) {
4595
      g_queryInfo.superQueryInfo.sqlCount = 0;
4596
    } else if (superSqls->type != cJSON_Array) {
4597
      errorPrint("%s() LN%d: failed to read json, super sqls not found\n",
4598
          __func__, __LINE__);
4599
      goto PARSE_OVER;
4600
    } else {
4601
      int superSqlSize = cJSON_GetArraySize(superSqls);
4602
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4603
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
4604
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4605 4606
        goto PARSE_OVER;
      }
4607

4608
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
4609
      for (int j = 0; j < superSqlSize; ++j) {
4610
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
4611
        if (sql == NULL) continue;
4612

4613
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
4614 4615 4616
        if (!sqlStr || sqlStr->type != cJSON_String
            || sqlStr->valuestring == NULL) {
          errorPrint("%s() LN%d, failed to read json, sql not found\n",
4617
              __func__, __LINE__);
4618 4619
          goto PARSE_OVER;
        }
4620
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring,
4621
            MAX_QUERY_SQL_LENGTH);
4622 4623

        cJSON *result = cJSON_GetObjectItem(sql, "result");
4624 4625 4626
        if (result != NULL && result->type == cJSON_String
            && result->valuestring != NULL){
          tstrncpy(g_queryInfo.superQueryInfo.result[j],
4627
              result->valuestring, MAX_FILE_NAME_LEN);
4628
        } else if (NULL == result) {
4629
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4630
        }  else {
4631
          errorPrint("%s() LN%d, failed to read json, sub query result file not found\n",
4632
              __func__, __LINE__);
4633
          goto PARSE_OVER;
4634
        }
4635 4636
      }
    }
H
hzcheng 已提交
4637 4638
  }

4639
  ret = true;
H
hzcheng 已提交
4640

4641
PARSE_OVER:
4642 4643
  return ret;
}
H
hzcheng 已提交
4644

4645
static bool getInfoFromJsonFile(char* file) {
4646
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
4647

4648 4649 4650 4651 4652
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
4653

4654
  bool  ret = false;
4655
  int   maxLen = 6400000;
4656 4657 4658 4659 4660 4661 4662
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
4663
  }
H
Hui Li 已提交
4664

4665 4666 4667
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
4668
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
4669 4670
    goto PARSE_OVER;
  }
H
Hui Li 已提交
4671

4672 4673 4674
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
4675
      g_args.test_mode = INSERT_TEST;
4676
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
4677
      g_args.test_mode = QUERY_TEST;
4678
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
4679
      g_args.test_mode = SUBSCRIBE_TEST;
4680
    } else {
4681
      printf("ERROR: failed to read json, filetype not support\n");
4682 4683 4684
      goto PARSE_OVER;
    }
  } else if (!filetype) {
4685
    g_args.test_mode = INSERT_TEST;
4686
  } else {
4687
    printf("ERROR: failed to read json, filetype not found\n");
4688 4689
    goto PARSE_OVER;
  }
H
hzcheng 已提交
4690

4691
  if (INSERT_TEST == g_args.test_mode) {
4692
    ret = getMetaFromInsertJsonFile(root);
4693
  } else if ((QUERY_TEST == g_args.test_mode)
4694
          || (SUBSCRIBE_TEST == g_args.test_mode)) {
4695 4696
    ret = getMetaFromQueryJsonFile(root);
  } else {
4697
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n",
4698
            __func__, __LINE__);
4699
    goto PARSE_OVER;
4700
  }
4701

4702
PARSE_OVER:
4703 4704
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
4705
  fclose(fp);
4706 4707
  return ret;
}
H
hzcheng 已提交
4708

4709
static int prepareSampleData() {
4710
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4711 4712
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
4713 4714 4715
        if (readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]) != 0) {
          return -1;
        }
S
Shuaiqiang Chang 已提交
4716
      }
4717 4718
    }
  }
4719 4720

  return 0;
4721 4722
}

4723
static void postFreeResource() {
4724
  tmfclose(g_fpOfInsertResult);
4725
  for (int i = 0; i < g_Dbs.dbCount; i++) {
S
Shuduo Sang 已提交
4726
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
4727 4728 4729
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
4730
      }
4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4743
    }
4744 4745
  }
}
S
Shuaiqiang Chang 已提交
4746

4747 4748 4749
static int getRowDataFromSample(
        char* dataBuf, int64_t maxLen, int64_t timestamp,
      SSuperTable* superTblInfo, int64_t* sampleUsePos) {
4750
  if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
4751
/*    int ret = readSampleFromCsvFileToMem(superTblInfo);
4752
    if (0 != ret) {
4753
      tmfree(superTblInfo->sampleDataBuf);
4754
      superTblInfo->sampleDataBuf = NULL;
4755 4756
      return -1;
    }
4757
*/
4758 4759
    *sampleUsePos = 0;
  }
S
Shuaiqiang Chang 已提交
4760

4761
  int    dataLen = 0;
4762

4763
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
4764
          "(%" PRId64 ", ", timestamp);
4765
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
4766
          "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
4767
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
S
Shuaiqiang Chang 已提交
4768

4769
  (*sampleUsePos)++;
4770

4771 4772
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4773

4774 4775
static int64_t generateStbRowData(
        SSuperTable* stbInfo,
4776 4777
        char* recBuf, int64_t timestamp)
{
4778
  int64_t   dataLen = 0;
4779
  char  *pstr = recBuf;
4780
  int64_t maxLen = MAX_DATA_SIZE;
4781

4782 4783
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
          "(%" PRId64 ",", timestamp);
4784

4785
  for (int i = 0; i < stbInfo->columnCount; i++) {
4786 4787 4788 4789
    if ((0 == strncasecmp(stbInfo->columns[i].dataType,
                    "BINARY", strlen("BINARY")))
            || (0 == strncasecmp(stbInfo->columns[i].dataType,
                    "NCHAR", strlen("NCHAR")))) {
4790
      if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
4791
        errorPrint( "binary or nchar length overflow, max size:%u\n",
4792
                (uint32_t)TSDB_MAX_BINARY_LEN);
4793
        return -1;
4794
      }
4795

4796 4797
      char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
      if (NULL == buf) {
4798
        errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
4799
        return -1;
4800 4801
      }
      rand_string(buf, stbInfo->columns[i].dataLen);
4802
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf);
4803
      tmfree(buf);
4804
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4805
                "INT", strlen("INT"))) {
4806
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4807
              "%d,", rand_int());
4808
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4809
                "BIGINT", strlen("BIGINT"))) {
4810
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4811
              "%"PRId64",", rand_bigint());
4812
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4813
                "FLOAT", strlen("FLOAT"))) {
4814
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4815
              "%f,", rand_float());
4816
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4817
                "DOUBLE", strlen("DOUBLE"))) {
4818
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4819
              "%f,", rand_double());
4820
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4821
                "SMALLINT", strlen("SMALLINT"))) {
4822
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4823
          "%d,", rand_smallint());
4824
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4825
          "TINYINT", strlen("TINYINT"))) {
4826
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4827
          "%d,", rand_tinyint());
4828
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4829
          "BOOL", strlen("BOOL"))) {
4830
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4831
          "%d,", rand_bool());
4832
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4833
          "TIMESTAMP", strlen("TIMESTAMP"))) {
4834
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4835
          "%"PRId64",", rand_bigint());
4836
    }  else {
4837
      errorPrint( "Not support data type: %s\n", stbInfo->columns[i].dataType);
4838
      return -1;
4839
    }
S
Shuaiqiang Chang 已提交
4840
  }
4841

4842
  dataLen -= 1;
4843
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")");
4844

4845
  verbosePrint("%s() LN%d, dataLen:%"PRId64"\n", __func__, __LINE__, dataLen);
4846 4847 4848
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

  return strlen(recBuf);
4849
}
S
Shuaiqiang Chang 已提交
4850

4851
static int64_t generateData(char *recBuf, char **data_type,
4852
        int64_t timestamp, int lenOfBinary) {
4853 4854
  memset(recBuf, 0, MAX_DATA_SIZE);
  char *pstr = recBuf;
4855 4856
  pstr += sprintf(pstr, "(%" PRId64, timestamp);

4857
  int columnCount = g_args.num_of_CPR;
4858

4859 4860
  for (int i = 0; i < columnCount; i++) {
    if (strcasecmp(data_type[i % columnCount], "TINYINT") == 0) {
4861
      pstr += sprintf(pstr, ",%d", rand_tinyint() );
4862
    } else if (strcasecmp(data_type[i % columnCount], "SMALLINT") == 0) {
4863
      pstr += sprintf(pstr, ",%d", rand_smallint());
4864
    } else if (strcasecmp(data_type[i % columnCount], "INT") == 0) {
4865
      pstr += sprintf(pstr, ",%d", rand_int());
4866
    } else if (strcasecmp(data_type[i % columnCount], "BIGINT") == 0) {
4867
      pstr += sprintf(pstr, ",%" PRId64, rand_bigint());
4868
    } else if (strcasecmp(data_type[i % columnCount], "TIMESTAMP") == 0) {
4869
      pstr += sprintf(pstr, ",%" PRId64, rand_bigint());
4870
    } else if (strcasecmp(data_type[i % columnCount], "FLOAT") == 0) {
4871
      pstr += sprintf(pstr, ",%10.4f", rand_float());
4872
    } else if (strcasecmp(data_type[i % columnCount], "DOUBLE") == 0) {
4873
      double t = rand_double();
4874
      pstr += sprintf(pstr, ",%20.8f", t);
4875
    } else if (strcasecmp(data_type[i % columnCount], "BOOL") == 0) {
4876
      bool b = rand_bool() & 1;
4877
      pstr += sprintf(pstr, ",%s", b ? "true" : "false");
4878
    } else if (strcasecmp(data_type[i % columnCount], "BINARY") == 0) {
4879 4880
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4881
      pstr += sprintf(pstr, ",\"%s\"", s);
4882
      free(s);
4883
    } else if (strcasecmp(data_type[i % columnCount], "NCHAR") == 0) {
4884 4885
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4886
      pstr += sprintf(pstr, ",\"%s\"", s);
4887 4888 4889
      free(s);
    }

4890
    if (strlen(recBuf) > MAX_DATA_SIZE) {
4891 4892 4893 4894 4895 4896 4897
      perror("column length too long, abort");
      exit(-1);
    }
  }

  pstr += sprintf(pstr, ")");

4898 4899
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

4900
  return (int32_t)strlen(recBuf);
4901 4902
}

4903
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
4904 4905
  char* sampleDataBuf = NULL;

4906
  sampleDataBuf = calloc(
4907
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
4908
  if (sampleDataBuf == NULL) {
4909
      errorPrint("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n",
4910 4911 4912 4913
              __func__, __LINE__,
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE,
              strerror(errno));
      return -1;
4914
  }
4915

4916 4917 4918 4919
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
4920 4921 4922 4923 4924
      errorPrint("%s() LN%d, read sample from csv file failed.\n",
          __func__, __LINE__);
      tmfree(sampleDataBuf);
      superTblInfo->sampleDataBuf = NULL;
      return -1;
H
Hui Li 已提交
4925
  }
4926

4927 4928 4929
  return 0;
}

4930
static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k)
4931
{
4932 4933
    int32_t affectedRows;
    SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4934

4935
    verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
S
Shuduo Sang 已提交
4936
            __func__, __LINE__, pThreadInfo->buffer);
S
Shuduo Sang 已提交
4937

4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957
    uint16_t iface;
    if (superTblInfo)
        iface = superTblInfo->iface;
    else
        iface = g_args.iface;

    debugPrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
            __func__, __LINE__,
            (g_args.iface==TAOSC_IFACE)?
            "taosc":(g_args.iface==REST_IFACE)?"rest":"stmt");

    switch(iface) {
        case TAOSC_IFACE:
            affectedRows = queryDbExec(
                    pThreadInfo->taos,
                    pThreadInfo->buffer, INSERT_TYPE, false);
            break;

        case REST_IFACE:
            if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port,
4958
                        pThreadInfo->buffer, pThreadInfo)) {
4959 4960 4961 4962 4963 4964 4965 4966
                affectedRows = -1;
                printf("========restful return fail, threadID[%d]\n",
                        pThreadInfo->threadID);
            } else {
                affectedRows = k;
            }
            break;

4967
#if STMT_IFACE_ENABLED == 1
4968 4969 4970 4971 4972 4973 4974 4975 4976
        case STMT_IFACE:
            debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt);
            if (0 != taos_stmt_execute(pThreadInfo->stmt)) {
                errorPrint("%s() LN%d, failied to execute insert statement\n",
                        __func__, __LINE__);
                exit(-1);
            }
            affectedRows = k;
            break;
4977
#endif
4978 4979 4980 4981 4982

        default:
            errorPrint("%s() LN%d: unknown insert mode: %d\n",
                    __func__, __LINE__, superTblInfo->iface);
            affectedRows = 0;
4983
    }
4984

4985
    return affectedRows;
4986 4987
}

4988 4989
static void getTableName(char *pTblName,
        threadInfo* pThreadInfo, uint64_t tableSeq)
4990
{
4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009
    SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
    if (superTblInfo) {
        if (AUTO_CREATE_SUBTBL != superTblInfo->autoCreateTable) {
            if (superTblInfo->childTblLimit > 0) {
                snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
                        superTblInfo->childTblName +
                        (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
            } else {
                verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n",
                        pThreadInfo->threadID, __func__, __LINE__,
                        pThreadInfo->start_table_from,
                        pThreadInfo->ntables, tableSeq);
                snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
                        superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
            }
        } else {
            snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"",
                    superTblInfo->childTblPrefix, tableSeq);
        }
5010
    } else {
5011 5012
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"",
                g_args.tb_prefix, tableSeq);
5013 5014 5015
    }
}

5016
static int32_t generateDataTailWithoutStb(
5017
        uint32_t batch, char* buffer,
5018
        int64_t remainderBufLen, int64_t insertRows,
5019
        uint64_t recordFrom, int64_t startTime,
5020
        /* int64_t *pSamplePos, */int64_t *dataLen) {
5021

5022
  uint64_t len = 0;
5023 5024
  char *pstr = buffer;

5025
  verbosePrint("%s() LN%d batch=%d\n", __func__, __LINE__, batch);
5026

5027
  int32_t k = 0;
5028
  for (k = 0; k < batch;) {
5029
    char data[MAX_DATA_SIZE];
5030 5031
    memset(data, 0, MAX_DATA_SIZE);

5032
    int64_t retLen = 0;
5033

5034 5035
    char **data_type = g_args.datatype;
    int lenOfBinary = g_args.len_of_binary;
5036

5037
    retLen = generateData(data, data_type,
5038
            startTime + getTSRandTail(
5039
                (int64_t) DEFAULT_TIMESTAMP_STEP, k,
5040 5041
                g_args.disorderRatio,
                g_args.disorderRange),
5042
            lenOfBinary);
5043

5044
    if (len > remainderBufLen)
5045
        break;
5046

5047 5048 5049 5050 5051
    pstr += sprintf(pstr, "%s", data);
    k++;
    len += retLen;
    remainderBufLen -= retLen;

5052
    verbosePrint("%s() LN%d len=%"PRIu64" k=%d \nbuffer=%s\n",
5053
            __func__, __LINE__, len, k, buffer);
5054

5055
    recordFrom ++;
5056

5057
    if (recordFrom >= insertRows) {
5058 5059 5060
      break;
    }
  }
5061

5062 5063 5064 5065
  *dataLen = len;
  return k;
}

5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082
static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq,
        int disorderRatio, int disorderRange)
{
    int64_t randTail = timeStampStep * seq;
    if (disorderRatio > 0) {
        int rand_num = taosRandom() % 100;
        if(rand_num < disorderRatio) {
            randTail = (randTail +
                    (taosRandom() % disorderRange + 1)) * (-1);
            debugPrint("rand data generated, back %"PRId64"\n", randTail);
        }
    }

    return randTail;
}

static int32_t generateStbDataTail(
5083
        SSuperTable* superTblInfo,
5084
        uint32_t batch, char* buffer,
5085
        int64_t remainderBufLen, int64_t insertRows,
5086
        uint64_t recordFrom, int64_t startTime,
5087 5088 5089 5090 5091
        int64_t *pSamplePos, int64_t *dataLen) {
  uint64_t len = 0;

  char *pstr = buffer;

5092 5093 5094 5095 5096 5097
  bool tsRand;
  if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) {
      tsRand = true;
  } else {
      tsRand = false;
  }
5098 5099
  verbosePrint("%s() LN%d batch=%u buflen=%"PRId64"\n",
          __func__, __LINE__, batch, remainderBufLen);
5100

5101
  int32_t k = 0;
5102 5103 5104 5105
  for (k = 0; k < batch;) {
    char data[MAX_DATA_SIZE];
    memset(data, 0, MAX_DATA_SIZE);

5106
    int64_t lenOfRow = 0;
5107

5108
    if (tsRand) {
5109
        lenOfRow = generateStbRowData(superTblInfo, data,
5110 5111 5112 5113 5114 5115
                startTime + getTSRandTail(
                    superTblInfo->timeStampStep, k,
                    superTblInfo->disorderRatio,
                    superTblInfo->disorderRange)
                );
    } else {
5116
        lenOfRow = getRowDataFromSample(
5117
                  data,
5118
                  (remainderBufLen < MAX_DATA_SIZE)?remainderBufLen:MAX_DATA_SIZE,
5119 5120 5121 5122
                  startTime + superTblInfo->timeStampStep * k,
                  superTblInfo,
                  pSamplePos);
    }
5123

5124
    if ((lenOfRow + 1) > remainderBufLen) {
5125
        break;
5126 5127
    }

5128
    pstr += snprintf(pstr , lenOfRow + 1, "%s", data);
5129
    k++;
5130 5131
    len += lenOfRow;
    remainderBufLen -= lenOfRow;
5132

5133
    verbosePrint("%s() LN%d len=%"PRIu64" k=%u \nbuffer=%s\n",
5134
            __func__, __LINE__, len, k, buffer);
5135

5136
    recordFrom ++;
5137

5138
    if (recordFrom >= insertRows) {
5139
      break;
5140
    }
5141
  }
5142

5143 5144 5145
  *dataLen = len;
  return k;
}
5146

5147 5148 5149

static int generateSQLHeadWithoutStb(char *tableName,
        char *dbName,
5150
        char *buffer, int remainderBufLen)
5151 5152
{
  int len;
5153 5154 5155

  char headBuf[HEAD_BUFF_LEN];

5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181
  len = snprintf(
          headBuf,
          HEAD_BUFF_LEN,
          "%s.%s values",
          dbName,
          tableName);

  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

  return len;
}

static int generateStbSQLHead(
        SSuperTable* superTblInfo,
        char *tableName, int32_t tableSeq,
        char *dbName,
        char *buffer, int remainderBufLen)
{
  int len;

  char headBuf[HEAD_BUFF_LEN];

  if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
5182 5183
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
5184
            tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq);
5185
      } else {
5186 5187 5188
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
5189 5190
      }
      if (NULL == tagsValBuf) {
5191
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n",
5192
            __func__, __LINE__);
5193 5194 5195
        return -1;
      }

5196 5197
      len = snprintf(
          headBuf,
5198 5199
                  HEAD_BUFF_LEN,
                  "%s.%s using %s.%s tags %s values",
5200
                  dbName,
5201
                  tableName,
5202
                  dbName,
5203 5204
                  superTblInfo->sTblName,
                  tagsValBuf);
5205 5206
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
5207 5208
      len = snprintf(
          headBuf,
5209 5210
                  HEAD_BUFF_LEN,
                  "%s.%s values",
5211
                  dbName,
5212
                  tableName);
5213
    } else {
5214 5215
      len = snprintf(
          headBuf,
5216 5217
                  HEAD_BUFF_LEN,
                  "%s.%s values",
5218
                  dbName,
5219
                  tableName);
5220 5221
  }

5222 5223 5224 5225 5226
  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

5227 5228 5229
  return len;
}

5230
static int32_t generateStbInterlaceData(
5231
        SSuperTable *superTblInfo,
5232 5233 5234
        char *tableName, uint32_t batchPerTbl,
        uint64_t i,
        uint32_t batchPerTblTimes,
5235
        uint64_t tableSeq,
5236
        threadInfo *pThreadInfo, char *buffer,
5237
        int64_t insertRows,
5238
        int64_t startTime,
5239
        uint64_t *pRemainderBufLen)
5240
{
5241
  assert(buffer);
5242 5243
  char *pstr = buffer;

5244 5245 5246
  int headLen = generateStbSQLHead(
          superTblInfo,
          tableName, tableSeq, pThreadInfo->db_name,
5247
          pstr, *pRemainderBufLen);
5248 5249 5250 5251 5252

  if (headLen <= 0) {
    return 0;
  }
  // generate data buffer
5253
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n",
5254
            pThreadInfo->threadID, __func__, __LINE__, i, buffer);
5255 5256 5257 5258

  pstr += headLen;
  *pRemainderBufLen -= headLen;

5259
  int64_t dataLen = 0;
5260

5261
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%u batchPerTbl = %u\n",
5262 5263
            pThreadInfo->threadID, __func__, __LINE__,
            i, batchPerTblTimes, batchPerTbl);
5264

5265
  if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
5266
      startTime = taosGetTimestamp(pThreadInfo->time_precision);
5267
  }
5268

5269
  int32_t k = generateStbDataTail(
5270 5271 5272 5273
            superTblInfo,
            batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
            startTime,
            &(pThreadInfo->samplePos), &dataLen);
5274 5275 5276 5277

  if (k == batchPerTbl) {
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
5278
  } else {
5279
    debugPrint("%s() LN%d, generated data tail: %u, not equal batch per table: %u\n",
5280 5281 5282 5283 5284 5285 5286 5287 5288 5289
            __func__, __LINE__, k, batchPerTbl);
    pstr -= headLen;
    pstr[0] = '\0';
    k = 0;
  }

  return k;
}

static int64_t generateInterlaceDataWithoutStb(
5290
        char *tableName, uint32_t batch,
5291 5292 5293
        uint64_t tableSeq,
        char *dbName, char *buffer,
        int64_t insertRows,
5294
        int64_t startTime,
5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312
        uint64_t *pRemainderBufLen)
{
  assert(buffer);
  char *pstr = buffer;

  int headLen = generateSQLHeadWithoutStb(
          tableName, dbName,
            pstr, *pRemainderBufLen);

  if (headLen <= 0) {
    return 0;
  }

  pstr += headLen;
  *pRemainderBufLen -= headLen;

  int64_t dataLen = 0;

5313 5314
  int32_t k = generateDataTailWithoutStb(
            batch, pstr, *pRemainderBufLen, insertRows, 0,
5315
            startTime,
5316
            &dataLen);
5317

5318
  if (k == batch) {
5319 5320 5321
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
  } else {
5322 5323
    debugPrint("%s() LN%d, generated data tail: %d, not equal batch per table: %u\n",
            __func__, __LINE__, k, batch);
5324 5325
    pstr -= headLen;
    pstr[0] = '\0';
5326
    k = 0;
5327 5328 5329 5330 5331
  }

  return k;
}

5332
#if STMT_IFACE_ENABLED == 1
5333
static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind,
5334
        char *dataType, int32_t dataLen, char **ptr, char *value)
5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345
{
    if (0 == strncasecmp(dataType,
                "BINARY", strlen("BINARY"))) {
        if (dataLen > TSDB_MAX_BINARY_LEN) {
            errorPrint( "binary length overflow, max size:%u\n",
                    (uint32_t)TSDB_MAX_BINARY_LEN);
            return -1;
        }
        char *bind_binary = (char *)*ptr;

        bind->buffer_type = TSDB_DATA_TYPE_BINARY;
5346 5347 5348 5349 5350 5351 5352 5353
        if (value) {
            strncpy(bind_binary, value, strlen(value));
            bind->buffer_length = strlen(bind_binary);
        } else {
            rand_string(bind_binary, dataLen);
            bind->buffer_length = dataLen;
        }

5354
        bind->length = &bind->buffer_length;
5355
        bind->buffer = bind_binary;
5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    } else if (0 == strncasecmp(dataType,
                "NCHAR", strlen("NCHAR"))) {
        if (dataLen > TSDB_MAX_BINARY_LEN) {
            errorPrint( "nchar length overflow, max size:%u\n",
                    (uint32_t)TSDB_MAX_BINARY_LEN);
            return -1;
        }
        char *bind_nchar = (char *)*ptr;

        bind->buffer_type = TSDB_DATA_TYPE_NCHAR;
5369 5370 5371 5372 5373 5374
        if (value) {
            strncpy(bind_nchar, value, strlen(value));
        } else {
            rand_string(bind_nchar, dataLen);
        }

5375 5376 5377 5378 5379 5380 5381 5382 5383 5384
        bind->buffer_length = strlen(bind_nchar);
        bind->buffer = bind_nchar;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    } else if (0 == strncasecmp(dataType,
                "INT", strlen("INT"))) {
        int32_t *bind_int = (int32_t *)*ptr;

5385 5386 5387 5388 5389
        if (value) {
            *bind_int = atoi(value);
        } else {
            *bind_int = rand_int();
        }
5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400
        bind->buffer_type = TSDB_DATA_TYPE_INT;
        bind->buffer_length = sizeof(int32_t);
        bind->buffer = bind_int;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    } else if (0 == strncasecmp(dataType,
                "BIGINT", strlen("BIGINT"))) {
        int64_t *bind_bigint = (int64_t *)*ptr;

5401 5402 5403 5404 5405
        if (value) {
            *bind_bigint = atoll(value);
        } else {
            *bind_bigint = rand_bigint();
        }
5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416
        bind->buffer_type = TSDB_DATA_TYPE_BIGINT;
        bind->buffer_length = sizeof(int64_t);
        bind->buffer = bind_bigint;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    }  else if (0 == strncasecmp(dataType,
                "FLOAT", strlen("FLOAT"))) {
        float   *bind_float = (float *) *ptr;

5417 5418 5419 5420 5421
        if (value) {
            *bind_float = (float)atof(value);
        } else {
            *bind_float = rand_float();
        }
5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432
        bind->buffer_type = TSDB_DATA_TYPE_FLOAT;
        bind->buffer_length = sizeof(float);
        bind->buffer = bind_float;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    }  else if (0 == strncasecmp(dataType,
                "DOUBLE", strlen("DOUBLE"))) {
        double  *bind_double = (double *)*ptr;

5433 5434 5435 5436 5437
        if (value) {
            *bind_double = atof(value);
        } else {
            *bind_double = rand_double();
        }
5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448
        bind->buffer_type = TSDB_DATA_TYPE_DOUBLE;
        bind->buffer_length = sizeof(double);
        bind->buffer = bind_double;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    }  else if (0 == strncasecmp(dataType,
                "SMALLINT", strlen("SMALLINT"))) {
        int16_t *bind_smallint = (int16_t *)*ptr;

5449 5450 5451 5452 5453
        if (value) {
            *bind_smallint = (int16_t)atoi(value);
        } else {
            *bind_smallint = rand_smallint();
        }
5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464
        bind->buffer_type = TSDB_DATA_TYPE_SMALLINT;
        bind->buffer_length = sizeof(int16_t);
        bind->buffer = bind_smallint;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    }  else if (0 == strncasecmp(dataType,
                "TINYINT", strlen("TINYINT"))) {
        int8_t  *bind_tinyint = (int8_t *)*ptr;

5465 5466 5467 5468 5469
        if (value) {
            *bind_tinyint = (int8_t)atoi(value);
        } else {
            *bind_tinyint = rand_tinyint();
        }
5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491
        bind->buffer_type = TSDB_DATA_TYPE_TINYINT;
        bind->buffer_length = sizeof(int8_t);
        bind->buffer = bind_tinyint;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;
        *ptr += bind->buffer_length;
    }  else if (0 == strncasecmp(dataType,
                "BOOL", strlen("BOOL"))) {
        int8_t  *bind_bool = (int8_t *)*ptr;

        *bind_bool = rand_bool();
        bind->buffer_type = TSDB_DATA_TYPE_BOOL;
        bind->buffer_length = sizeof(int8_t);
        bind->buffer = bind_bool;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    }  else if (0 == strncasecmp(dataType,
                "TIMESTAMP", strlen("TIMESTAMP"))) {
        int64_t *bind_ts2 = (int64_t *) *ptr;

5492 5493 5494 5495 5496
        if (value) {
            *bind_ts2 = atoll(value);
        } else {
            *bind_ts2 = rand_bigint();
        }
5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513
        bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP;
        bind->buffer_length = sizeof(int64_t);
        bind->buffer = bind_ts2;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    }  else {
        errorPrint( "No support data type: %s\n",
                dataType);
        return -1;
    }

    return 0;
}

static int32_t prepareStmtWithoutStb(
5514
        TAOS_STMT *stmt,
5515 5516 5517 5518 5519
        char *tableName,
        uint32_t batch,
        int64_t insertRows,
        int64_t recordFrom,
        int64_t startTime)
5520
{
5521 5522 5523 5524 5525 5526 5527 5528
    int ret = taos_stmt_set_tbname(stmt, tableName);
    if (ret != 0) {
        errorPrint("failed to execute taos_stmt_set_tbname(%s). return 0x%x. reason: %s\n",
                tableName, ret, taos_errstr(NULL));
        return ret;
    }

    char **data_type = g_args.datatype;
5529

5530 5531 5532 5533 5534
    char *bindArray = malloc(sizeof(TAOS_BIND) * (g_args.num_of_CPR + 1));
    if (bindArray == NULL) {
        errorPrint("Failed to allocate %d bind params\n",
                (g_args.num_of_CPR + 1));
        return -1;
5535 5536
    }

5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561
    int32_t k = 0;
    for (k = 0; k < batch;) {
        /* columnCount + 1 (ts) */
        char data[MAX_DATA_SIZE];
        memset(data, 0, MAX_DATA_SIZE);

        char *ptr = data;
        TAOS_BIND *bind = (TAOS_BIND *)(bindArray + 0);

        int64_t *bind_ts;

        bind_ts = (int64_t *)ptr;
        bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP;
        *bind_ts = startTime + getTSRandTail(
                (int64_t)DEFAULT_TIMESTAMP_STEP, k,
                g_args.disorderRatio,
                g_args.disorderRange);
        bind->buffer_length = sizeof(int64_t);
        bind->buffer = bind_ts;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        ptr += bind->buffer_length;

        for (int i = 0; i < g_args.num_of_CPR; i ++) {
5562 5563
            bind = (TAOS_BIND *)((char *)bindArray
                    + (sizeof(TAOS_BIND) * (i + 1)));
5564 5565 5566 5567
            if ( -1 == prepareStmtBindArrayByType(
                        bind,
                        data_type[i],
                        g_args.len_of_binary,
5568
                        &ptr, NULL)) {
5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582
                return -1;
            }
        }
        taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray);
        // if msg > 3MB, break
        taos_stmt_add_batch(stmt);

        k++;
        recordFrom ++;
        if (recordFrom >= insertRows) {
            break;
        }
    }

5583
    free(bindArray);
5584 5585 5586
    return k;
}

5587 5588
static int32_t prepareStbStmt(
        SSuperTable *stbInfo,
5589 5590 5591 5592
        TAOS_STMT *stmt,
        char *tableName, uint32_t batch,
        uint64_t insertRows,
        uint64_t recordFrom,
5593 5594
        int64_t startTime,
        int64_t *pSamplePos)
5595 5596 5597 5598 5599
{
    int ret = taos_stmt_set_tbname(stmt, tableName);
    if (ret != 0) {
        errorPrint("failed to execute taos_stmt_set_tbname(%s). return 0x%x. reason: %s\n",
                tableName, ret, taos_errstr(NULL));
5600 5601 5602
        return ret;
    }

5603
    char *bindArray = malloc(sizeof(TAOS_BIND) * (stbInfo->columnCount + 1));
5604
    if (bindArray == NULL) {
5605 5606
        errorPrint("%s() LN%d, Failed to allocate %d bind params\n",
                __func__, __LINE__, (stbInfo->columnCount + 1));
5607 5608 5609
        return -1;
    }

5610
    bool sourceRand;
5611
    if (0 == strncasecmp(stbInfo->dataSource, "rand", strlen("rand"))) {
5612
        sourceRand = true;
5613
    } else {
5614 5615 5616 5617 5618 5619 5620 5621 5622
        sourceRand = false;     // from sample data file
    }

    char *bindBuffer = malloc(g_args.len_of_binary);
    if (bindBuffer == NULL) {
        errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n",
                __func__, __LINE__, g_args.len_of_binary);
        free(bindArray);
        return -1;
5623
    }
5624 5625

    uint32_t k;
5626
    for (k = 0; k < batch;) {
5627
        /* columnCount + 1 (ts) */
5628 5629 5630 5631 5632
        char data[MAX_DATA_SIZE];
        memset(data, 0, MAX_DATA_SIZE);

        char *ptr = data;
        TAOS_BIND *bind = (TAOS_BIND *)(bindArray + 0);
5633

5634 5635 5636 5637
        int64_t *bind_ts;

        bind_ts = (int64_t *)ptr;
        bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP;
5638
        if (sourceRand) {
5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652
            *bind_ts = startTime + getTSRandTail(
                    stbInfo->timeStampStep, k,
                    stbInfo->disorderRatio,
                    stbInfo->disorderRange);
        } else {
            *bind_ts = startTime + stbInfo->timeStampStep * k;
        }
        bind->buffer_length = sizeof(int64_t);
        bind->buffer = bind_ts;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        ptr += bind->buffer_length;

5653
        int cursor = 0;
5654 5655
        for (int i = 0; i < stbInfo->columnCount; i ++) {
            bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * (i + 1)));
5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692

            if (sourceRand) {
                if ( -1 == prepareStmtBindArrayByType(
                            bind,
                            stbInfo->columns[i].dataType,
                            stbInfo->columns[i].dataLen,
                            &ptr,
                            NULL)) {
                    free(bindArray);
                    free(bindBuffer);
                    return -1;
                }
            } else {
                char *restStr = stbInfo->sampleDataBuf + cursor;
                int lengthOfRest = strlen(restStr);

                int index = 0;
                for (index = 0; index < lengthOfRest; index ++) {
                    if (restStr[index] == ',') {
                        break;
                    }
                }

                memset(bindBuffer, 0, g_args.len_of_binary);
                strncpy(bindBuffer, restStr, index);
                cursor += index + 1; // skip ',' too

                if ( -1 == prepareStmtBindArrayByType(
                            bind,
                            stbInfo->columns[i].dataType,
                            stbInfo->columns[i].dataLen,
                            &ptr,
                            bindBuffer)) {
                    free(bindArray);
                    free(bindBuffer);
                    return -1;
                }
5693 5694
            }
        }
5695
        taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray);
5696
        // if msg > 3MB, break
5697
        taos_stmt_add_batch(stmt);
5698

5699 5700
        k++;
        recordFrom ++;
5701 5702 5703 5704

        if (!sourceRand) {
            (*pSamplePos) ++;
        }
5705 5706 5707 5708
        if (recordFrom >= insertRows) {
            break;
        }
    }
5709

5710
    free(bindBuffer);
5711
    free(bindArray);
5712 5713
    return k;
}
5714
#endif
5715 5716

static int32_t generateStbProgressiveData(
5717
        SSuperTable *superTblInfo,
5718
        char *tableName,
5719
        int64_t tableSeq,
5720
        char *dbName, char *buffer,
5721
        int64_t insertRows,
5722
        uint64_t recordFrom, int64_t startTime, int64_t *pSamplePos,
5723
        int64_t *pRemainderBufLen)
5724 5725
{
  assert(buffer != NULL);
5726
  char *pstr = buffer;
5727

5728
  memset(buffer, 0, *pRemainderBufLen);
5729

5730
  int64_t headLen = generateStbSQLHead(
5731
              superTblInfo,
5732
          tableName, tableSeq, dbName,
5733
          buffer, *pRemainderBufLen);
5734 5735 5736 5737

  if (headLen <= 0) {
    return 0;
  }
5738
  pstr += headLen;
5739
  *pRemainderBufLen -= headLen;
5740

5741
  int64_t dataLen;
5742

5743
  return generateStbDataTail(superTblInfo,
5744
          g_args.num_of_RPR, pstr, *pRemainderBufLen,
5745
          insertRows, recordFrom,
5746 5747
          startTime,
          pSamplePos, &dataLen);
5748 5749
}

5750
static int32_t generateProgressiveDataWithoutStb(
5751
        char *tableName,
5752
        /* int64_t tableSeq, */
5753 5754
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
5755
        uint64_t recordFrom, int64_t startTime, /*int64_t *pSamplePos, */
5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775
        int64_t *pRemainderBufLen)
{
  assert(buffer != NULL);
  char *pstr = buffer;

  memset(buffer, 0, *pRemainderBufLen);

  int64_t headLen = generateSQLHeadWithoutStb(
          tableName, pThreadInfo->db_name,
          buffer, *pRemainderBufLen);

  if (headLen <= 0) {
    return 0;
  }
  pstr += headLen;
  *pRemainderBufLen -= headLen;

  int64_t dataLen;

  return generateDataTailWithoutStb(
5776
          g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, recordFrom,
5777 5778
          startTime,
          /*pSamplePos, */&dataLen);
5779
}
5780

5781 5782
static void printStatPerThread(threadInfo *pThreadInfo)
{
5783
    fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n",
5784 5785 5786 5787 5788 5789
          pThreadInfo->threadID,
          pThreadInfo->totalInsertRows,
          pThreadInfo->totalAffectedRows,
          (double)(pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay/1000.0)));
}

5790
// sync write interlace data
5791
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
5792
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
5793 5794
         pThreadInfo->threadID, __func__, __LINE__);

5795
  int64_t insertRows;
5796
  uint32_t interlaceRows;
5797 5798 5799
  uint64_t maxSqlLen;
  int64_t nTimeStampStep;
  uint64_t insert_interval;
5800 5801

  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5802

5803 5804 5805 5806 5807 5808 5809 5810 5811
  if (superTblInfo) {
    insertRows = superTblInfo->insertRows;

    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
5812 5813 5814
    maxSqlLen = superTblInfo->maxSqlLen;
    nTimeStampStep = superTblInfo->timeStampStep;
    insert_interval = superTblInfo->insertInterval;
5815 5816 5817
  } else {
    insertRows = g_args.num_of_DPT;
    interlaceRows = g_args.interlace_rows;
5818 5819 5820
    maxSqlLen = g_args.max_sql_len;
    nTimeStampStep = DEFAULT_TIMESTAMP_STEP;
    insert_interval = g_args.insert_interval;
5821
  }
5822

5823 5824 5825 5826 5827
  debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n",
          pThreadInfo->threadID, __func__, __LINE__,
          pThreadInfo->start_table_from,
          pThreadInfo->ntables, insertRows);

5828
  if (interlaceRows > insertRows)
H
Haojun Liao 已提交
5829
    interlaceRows = insertRows;
5830

5831 5832
  if (interlaceRows > g_args.num_of_RPR)
    interlaceRows = g_args.num_of_RPR;
5833

5834 5835
  uint32_t batchPerTbl = interlaceRows;
  uint32_t batchPerTblTimes;
5836

5837 5838 5839
  if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) {
    batchPerTblTimes =
        g_args.num_of_RPR / interlaceRows;
5840
  } else {
5841
    batchPerTblTimes = 1;
5842 5843
  }

S
Shuduo Sang 已提交
5844 5845
  pThreadInfo->buffer = calloc(maxSqlLen, 1);
  if (NULL == pThreadInfo->buffer) {
5846
    errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5847
              __func__, __LINE__, maxSqlLen, strerror(errno));
5848 5849 5850 5851 5852 5853
    return NULL;
  }

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

5854 5855
  uint64_t st = 0;
  uint64_t et = UINT64_MAX;
5856

5857 5858 5859
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5860

5861
  uint64_t tableSeq = pThreadInfo->start_table_from;
5862 5863
  int64_t startTime = pThreadInfo->start_time;

5864
  uint64_t generatedRecPerTbl = 0;
5865
  bool flagSleep = true;
5866
  uint64_t sleepTimeTotal = 0;
5867

5868
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
5869
    if ((flagSleep) && (insert_interval)) {
5870 5871
        st = taosGetTimestampMs();
        flagSleep = false;
5872 5873
    }
    // generate data
S
Shuduo Sang 已提交
5874
    memset(pThreadInfo->buffer, 0, maxSqlLen);
5875
    uint64_t remainderBufLen = maxSqlLen;
5876

S
Shuduo Sang 已提交
5877
    char *pstr = pThreadInfo->buffer;
5878

5879 5880
    int len = snprintf(pstr,
            strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO);
5881 5882 5883
    pstr += len;
    remainderBufLen -= len;

5884
    uint32_t recOfBatch = 0;
5885

5886
    for (uint32_t i = 0; i < batchPerTblTimes; i ++) {
5887 5888
        char tableName[TSDB_TABLE_NAME_LEN];

5889
      getTableName(tableName, pThreadInfo, tableSeq);
5890 5891
      if (0 == strlen(tableName)) {
        errorPrint("[%d] %s() LN%d, getTableName return null\n",
5892
            pThreadInfo->threadID, __func__, __LINE__);
S
Shuduo Sang 已提交
5893
        free(pThreadInfo->buffer);
5894 5895
        return NULL;
      }
5896

5897
      uint64_t oldRemainderLen = remainderBufLen;
5898

5899
      int32_t generated;
5900
      if (superTblInfo) {
5901
          if (superTblInfo->iface == STMT_IFACE) {
5902
#if STMT_IFACE_ENABLED == 1
5903 5904
              generated = prepareStbStmt(
                      superTblInfo,
5905 5906 5907 5908 5909
                      pThreadInfo->stmt,
                      tableName,
                      batchPerTbl,
                      insertRows, i,
                      startTime,
5910
                      &(pThreadInfo->samplePos));
5911 5912 5913
#else
              generated = -1;
#endif
5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924
          } else {
              generated = generateStbInterlaceData(
                      superTblInfo,
                      tableName, batchPerTbl, i,
                      batchPerTblTimes,
                      tableSeq,
                      pThreadInfo, pstr,
                      insertRows,
                      startTime,
                      &remainderBufLen);
          }
5925
      } else {
5926 5927 5928 5929 5930
          if (g_args.iface == STMT_IFACE) {
              debugPrint("[%d] %s() LN%d, tableName:%s, batch:%d startTime:%"PRId64"\n",
                      pThreadInfo->threadID,
                      __func__, __LINE__,
                      tableName, batchPerTbl, startTime);
5931
#if STMT_IFACE_ENABLED == 1
5932 5933 5934 5935 5936
              generated = prepareStmtWithoutStb(
                      pThreadInfo->stmt, tableName,
                      batchPerTbl,
                      insertRows, i,
                      startTime);
5937 5938 5939
#else
              generated = -1;
#endif
5940 5941 5942 5943 5944 5945 5946 5947 5948
          } else {
              generated = generateInterlaceDataWithoutStb(
                      tableName, batchPerTbl,
                      tableSeq,
                      pThreadInfo->db_name, pstr,
                      insertRows,
                      startTime,
                      &remainderBufLen);
          }
5949
      }
5950

5951
      debugPrint("[%d] %s() LN%d, generated records is %d\n",
5952
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5953
      if (generated < 0) {
5954
        errorPrint("[%d] %s() LN%d, generated records is %d\n",
5955
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5956
        goto free_of_interlace;
5957 5958
      } else if (generated == 0) {
        break;
5959 5960
      }

5961
      tableSeq ++;
5962
      recOfBatch += batchPerTbl;
5963

5964
      pstr += (oldRemainderLen - remainderBufLen);
5965
      pThreadInfo->totalInsertRows += batchPerTbl;
5966

5967
      verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n",
5968 5969
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
5970

5971
      if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
5972 5973 5974
            // turn to first table
            tableSeq = pThreadInfo->start_table_from;
            generatedRecPerTbl += batchPerTbl;
5975

5976 5977
            startTime = pThreadInfo->start_time
              + generatedRecPerTbl * nTimeStampStep;
5978

5979 5980 5981
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;
5982

5983
            int64_t remainRows = insertRows - generatedRecPerTbl;
5984 5985
            if ((remainRows > 0) && (batchPerTbl > remainRows))
              batchPerTbl = remainRows;
5986

5987 5988 5989
            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
      }
5990

5991
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n",
5992 5993
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);
5994 5995

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
5996 5997 5998
        break;
    }

5999
    verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRIu64"\n",
6000 6001
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
6002
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
S
Shuduo Sang 已提交
6003
           pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer);
6004

6005
    startTs = taosGetTimestampMs();
6006

6007
    if (recOfBatch == 0) {
6008 6009 6010 6011 6012 6013 6014 6015
        errorPrint("[%d] %s() LN%d Failed to insert records of batch %d\n",
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl);
        errorPrint("\tIf the batch is %d, the length of the SQL to insert a row must be less then %"PRId64"\n",
                batchPerTbl, maxSqlLen / batchPerTbl);
        errorPrint("\tPlease check if the buffer length(%"PRId64") or batch(%d) is set with proper value!\n",
                maxSqlLen, batchPerTbl);
        goto free_of_interlace;
6016
    }
S
Shuduo Sang 已提交
6017
    int64_t affectedRows = execInsert(pThreadInfo, recOfBatch);
6018

6019
    endTs = taosGetTimestampMs();
6020 6021
    uint64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n",
6022
            __func__, __LINE__, delay);
6023 6024 6025
    verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
6026 6027 6028 6029 6030 6031

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

6032
    if (recOfBatch != affectedRows) {
6033
        errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %"PRId64"\n%s\n",
6034
                pThreadInfo->threadID, __func__, __LINE__,
S
Shuduo Sang 已提交
6035
                recOfBatch, affectedRows, pThreadInfo->buffer);
6036
        goto free_of_interlace;
6037
    }
6038

6039
    pThreadInfo->totalAffectedRows += affectedRows;
6040

6041 6042
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
6043
      printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n",
6044 6045 6046
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
6047 6048
      lastPrintTime = currentPrintTime;
    }
6049

6050
    if ((insert_interval) && flagSleep) {
6051
      et = taosGetTimestampMs();
6052

6053
      if (insert_interval > (et - st) ) {
S
Shuduo Sang 已提交
6054 6055
        uint64_t sleepTime = insert_interval - (et -st);
        performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n",
6056
                    __func__, __LINE__, sleepTime);
6057 6058 6059
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
6060
    }
6061 6062
  }

6063
free_of_interlace:
S
Shuduo Sang 已提交
6064
  tmfree(pThreadInfo->buffer);
6065
  printStatPerThread(pThreadInfo);
6066 6067 6068
  return NULL;
}

6069
// sync insertion progressive data
6070
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
6071
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
6072

6073
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
6074
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
6075 6076 6077 6078 6079 6080
  int64_t timeStampStep =
      superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
  int64_t insertRows =
        (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
  verbosePrint("%s() LN%d insertRows=%"PRId64"\n",
            __func__, __LINE__, insertRows);
6081

S
Shuduo Sang 已提交
6082 6083
  pThreadInfo->buffer = calloc(maxSqlLen, 1);
  if (NULL == pThreadInfo->buffer) {
6084
    errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n",
6085 6086
              maxSqlLen,
              strerror(errno));
6087 6088
    return NULL;
  }
6089

6090 6091 6092
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
6093

6094 6095
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
6096

6097
  pThreadInfo->samplePos = 0;
6098

6099 6100 6101
  for (uint64_t tableSeq = pThreadInfo->start_table_from;
          tableSeq <= pThreadInfo->end_table_to;
          tableSeq ++) {
6102
    int64_t start_time = pThreadInfo->start_time;
6103

6104
    for (uint64_t i = 0; i < insertRows;) {
6105 6106
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
6107
      verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n",
6108 6109
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);
6110 6111 6112 6113 6114 6115
      if (0 == strlen(tableName)) {
        errorPrint("[%d] %s() LN%d, getTableName return null\n",
            pThreadInfo->threadID, __func__, __LINE__);
        free(pThreadInfo->buffer);
        return NULL;
      }
6116

6117
      int64_t remainderBufLen = maxSqlLen;
S
Shuduo Sang 已提交
6118
      char *pstr = pThreadInfo->buffer;
6119

6120 6121
      int len = snprintf(pstr,
              strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO);
6122 6123 6124 6125

      pstr += len;
      remainderBufLen -= len;

6126
      int32_t generated;
6127
      if (superTblInfo) {
6128
          if (superTblInfo->iface == STMT_IFACE) {
6129
#if STMT_IFACE_ENABLED == 1
6130 6131
              generated = prepareStbStmt(
                      superTblInfo,
6132
                      pThreadInfo->stmt,
6133 6134
                      tableName,
                      g_args.num_of_RPR,
6135 6136
                      insertRows, i, start_time,
                      &(pThreadInfo->samplePos));
6137 6138 6139
#else
              generated = -1;
#endif
6140 6141 6142 6143 6144 6145 6146 6147
          } else {
              generated = generateStbProgressiveData(
                      superTblInfo,
                      tableName, tableSeq, pThreadInfo->db_name, pstr,
                      insertRows, i, start_time,
                      &(pThreadInfo->samplePos),
                      &remainderBufLen);
          }
6148
      } else {
6149
          if (g_args.iface == STMT_IFACE) {
6150
#if STMT_IFACE_ENABLED == 1
6151 6152 6153 6154 6155 6156
              generated = prepareStmtWithoutStb(
                      pThreadInfo->stmt,
                      tableName,
                      g_args.num_of_RPR,
                      insertRows, i,
                      start_time);
6157 6158 6159
#else
              generated = -1;
#endif
6160 6161 6162 6163 6164 6165 6166 6167 6168
          } else {
              generated = generateProgressiveDataWithoutStb(
                      tableName,
                      /*  tableSeq, */
                      pThreadInfo, pstr, insertRows,
                      i, start_time,
                      /* &(pThreadInfo->samplePos), */
                      &remainderBufLen);
          }
6169
      }
6170 6171 6172
      if (generated > 0)
        i += generated;
      else
6173
        goto free_of_progressive;
6174

6175
      start_time +=  generated * timeStampStep;
6176
      pThreadInfo->totalInsertRows += generated;
6177

6178
      startTs = taosGetTimestampMs();
6179

6180
      int32_t affectedRows = execInsert(pThreadInfo, generated);
6181

6182
      endTs = taosGetTimestampMs();
6183
      uint64_t delay = endTs - startTs;
6184
      performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n",
6185
              __func__, __LINE__, delay);
6186
      verbosePrint("[%d] %s() LN%d affectedRows=%d\n",
6187 6188
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
6189

6190 6191 6192 6193
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
6194

6195
      if (affectedRows < 0) {
6196
        errorPrint("%s() LN%d, affected rows: %d\n",
6197 6198 6199
                __func__, __LINE__, affectedRows);
        goto free_of_progressive;
      }
6200 6201 6202

      pThreadInfo->totalAffectedRows += affectedRows;

6203 6204
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
6205
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
6206 6207 6208
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
6209 6210 6211
        lastPrintTime = currentPrintTime;
      }

6212
      if (i >= insertRows)
6213 6214 6215
        break;
    }   // num_of_DPT

6216 6217
    if ((g_args.verbose_print) &&
      (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) &&
6218 6219
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
6220
          verbosePrint("%s() LN%d samplePos=%"PRId64"\n",
6221
                  __func__, __LINE__, pThreadInfo->samplePos);
6222
    }
6223
  } // tableSeq
6224

6225
free_of_progressive:
S
Shuduo Sang 已提交
6226
  tmfree(pThreadInfo->buffer);
6227
  printStatPerThread(pThreadInfo);
6228
  return NULL;
H
Hui Li 已提交
6229 6230
}

6231 6232
static void* syncWrite(void *sarg) {

6233 6234
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
6235

6236
  uint32_t interlaceRows;
6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247

  if (superTblInfo) {
    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    interlaceRows = g_args.interlace_rows;
  }
6248

6249
  if (interlaceRows > 0) {
6250
    // interlace mode
6251
    return syncWriteInterlace(pThreadInfo);
6252 6253
  } else {
    // progressive mode
6254
    return syncWriteProgressive(pThreadInfo);
6255 6256 6257
  }
}

6258
static void callBack(void *param, TAOS_RES *res, int code) {
6259 6260
  threadInfo* pThreadInfo = (threadInfo*)param;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
6261

6262 6263
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
6264
  if (insert_interval) {
6265 6266 6267
    pThreadInfo->et = taosGetTimestampMs();
    if ((pThreadInfo->et - pThreadInfo->st) < insert_interval) {
      taosMsleep(insert_interval - (pThreadInfo->et - pThreadInfo->st)); // ms
6268
    }
H
Hui Li 已提交
6269
  }
6270

6271
  char *buffer = calloc(1, pThreadInfo->superTblInfo->maxSqlLen);
6272
  char data[MAX_DATA_SIZE];
6273
  char *pstr = buffer;
6274 6275
  pstr += sprintf(pstr, "insert into %s.%s%"PRId64" values",
          pThreadInfo->db_name, pThreadInfo->tb_prefix,
6276
          pThreadInfo->start_table_from);
6277 6278 6279 6280 6281 6282 6283
//  if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
  if (pThreadInfo->counter >= g_args.num_of_RPR) {
    pThreadInfo->start_table_from++;
    pThreadInfo->counter = 0;
  }
  if (pThreadInfo->start_table_from > pThreadInfo->end_table_to) {
    tsem_post(&pThreadInfo->lock_sem);
6284 6285 6286 6287
    free(buffer);
    taos_free_result(res);
    return;
  }
6288

6289
  for (int i = 0; i < g_args.num_of_RPR; i++) {
6290
    int rand_num = taosRandom() % 100;
6291
    if (0 != pThreadInfo->superTblInfo->disorderRatio
6292
            && rand_num < pThreadInfo->superTblInfo->disorderRatio) {
6293 6294 6295
      int64_t d = pThreadInfo->lastTs
          - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1);
      generateStbRowData(pThreadInfo->superTblInfo, data, d);
6296
    } else {
6297 6298
      generateStbRowData(pThreadInfo->superTblInfo,
              data, pThreadInfo->lastTs += 1000);
H
Hui Li 已提交
6299
    }
6300
    pstr += sprintf(pstr, "%s", data);
6301
    pThreadInfo->counter++;
H
Hui Li 已提交
6302

6303
    if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
6304
      break;
H
Hui Li 已提交
6305 6306
    }
  }
6307

6308
  if (insert_interval) {
6309
    pThreadInfo->st = taosGetTimestampMs();
6310
  }
6311
  taos_query_a(pThreadInfo->taos, buffer, callBack, pThreadInfo);
6312
  free(buffer);
H
Hui Li 已提交
6313

6314
  taos_free_result(res);
H
Hui Li 已提交
6315 6316
}

6317
static void *asyncWrite(void *sarg) {
6318 6319
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
6320

6321 6322 6323
  pThreadInfo->st = 0;
  pThreadInfo->et = 0;
  pThreadInfo->lastTs = pThreadInfo->start_time;
6324

6325
  int insert_interval =
6326
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
6327
  if (insert_interval) {
6328
    pThreadInfo->st = taosGetTimestampMs();
H
Hui Li 已提交
6329
  }
6330
  taos_query_a(pThreadInfo->taos, "show databases", callBack, pThreadInfo);
6331

6332
  tsem_wait(&(pThreadInfo->lock_sem));
H
Hui Li 已提交
6333 6334 6335 6336

  return NULL;
}

6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362
static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in *serv_addr)
{
  uint16_t rest_port = port + TSDB_PORT_HTTP;
  struct hostent *server = gethostbyname(host);
  if ((server == NULL) || (server->h_addr == NULL)) {
    errorPrint("%s", "ERROR, no such host");
    return -1;
  }

  debugPrint("h_name: %s\nh_addr=%p\nh_addretype: %s\nh_length: %d\n",
            server->h_name,
            server->h_addr,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

  memset(serv_addr, 0, sizeof(struct sockaddr_in));
  serv_addr->sin_family = AF_INET;
  serv_addr->sin_port = htons(rest_port);
#ifdef WINDOWS
  serv_addr->sin_addr.s_addr = inet_addr(host);
#else
  memcpy(&(serv_addr->sin_addr.s_addr), server->h_addr, server->h_length);
#endif
  return 0;
}

6363
static void startMultiThreadInsertData(int threads, char* db_name,
6364
        char* precision,SSuperTable* superTblInfo) {
6365

6366 6367 6368 6369
  int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
  if (0 != precision[0]) {
    if (0 == strncasecmp(precision, "ms", 2)) {
      timePrec = TSDB_TIME_PRECISION_MILLI;
6370
    } else if (0 == strncasecmp(precision, "us", 2)) {
6371
      timePrec = TSDB_TIME_PRECISION_MICRO;
6372 6373 6374
    } else if (0 == strncasecmp(precision, "ns", 2)) {
      timePrec = TSDB_TIME_PRECISION_NANO;
    } else {
6375
      errorPrint("Not support precision: %s\n", precision);
6376 6377 6378 6379
      exit(-1);
    }
  }

6380
  int64_t start_time;
6381 6382
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
6383
        start_time = taosGetTimestamp(timePrec);
6384
    } else {
6385
      if (TSDB_CODE_SUCCESS != taosParseTime(
6386 6387 6388 6389 6390
        superTblInfo->startTimestamp,
        &start_time,
        strlen(superTblInfo->startTimestamp),
        timePrec, 0)) {
          ERROR_EXIT("failed to parse time!\n");
6391
      }
6392
    }
6393
  } else {
6394
     start_time = 1500000000000;
6395 6396
  }

6397
  int64_t start = taosGetTimestampMs();
6398

6399
  // read sample data from file first
6400
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
6401
              "sample", strlen("sample")))) {
6402
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
6403
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
6404
              __func__, __LINE__);
6405 6406 6407 6408
      exit(-1);
    }
  }

6409
  TAOS* taos0 = taos_connect(
6410 6411
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
6412
  if (NULL == taos0) {
6413
    errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
6414
                __func__, __LINE__, taos_errstr(NULL));
6415 6416 6417
    exit(-1);
  }

S
Shuduo Sang 已提交
6418
  int64_t ntables = 0;
6419
  uint64_t tableFrom;
6420

6421
  if (superTblInfo) {
6422 6423
    int64_t limit;
    uint64_t offset;
6424

H
Haojun Liao 已提交
6425
    if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) &&
6426
            ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) {
H
Haojun Liao 已提交
6427
      printf("WARNING: offset and limit will not be used since the child tables not exists!\n");
6428 6429
    }

6430
    if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) {
6431 6432
      if ((superTblInfo->childTblLimit < 0)
          || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit)
6433
            > (superTblInfo->childTblCount))) {
6434 6435 6436 6437 6438 6439
        superTblInfo->childTblLimit =
            superTblInfo->childTblCount - superTblInfo->childTblOffset;
      }

      offset = superTblInfo->childTblOffset;
      limit = superTblInfo->childTblLimit;
6440
    } else {
6441 6442
      limit = superTblInfo->childTblCount;
      offset = 0;
6443 6444
    }

6445
    ntables = limit;
6446
    tableFrom = offset;
6447

6448 6449 6450 6451
    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
        && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
            > superTblInfo->childTblCount)) {
      printf("WARNING: specified offset + limit > child table count!\n");
6452
      prompt();
6453 6454 6455
    }

    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
6456
            && (0 == superTblInfo->childTblLimit)) {
6457
      printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
6458
      prompt();
6459 6460
    }

6461
    superTblInfo->childTblName = (char*)calloc(1,
6462
        limit * TSDB_TABLE_NAME_LEN);
6463
    if (superTblInfo->childTblName == NULL) {
6464
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
6465
      taos_close(taos0);
6466 6467 6468
      exit(-1);
    }

6469
    int64_t childTblCount;
6470
    getChildNameOfSuperTableWithLimitAndOffset(
6471
        taos0,
6472 6473 6474 6475
        db_name, superTblInfo->sTblName,
        &superTblInfo->childTblName, &childTblCount,
        limit,
        offset);
6476 6477
  } else {
    ntables = g_args.num_of_tables;
6478
    tableFrom = 0;
6479
  }
6480

6481
  taos_close(taos0);
6482

S
Shuduo Sang 已提交
6483
  int64_t a = ntables / threads;
6484 6485 6486 6487 6488
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

S
Shuduo Sang 已提交
6489
  int64_t b = 0;
6490 6491 6492 6493
  if (threads != 0) {
    b = ntables % threads;
  }

6494
  if ((superTblInfo)
6495 6496 6497
      && (superTblInfo->iface == REST_IFACE)) {
      if (convertHostToServAddr(
                  g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) {
6498 6499
        exit(-1);
      }
6500 6501
  }

6502 6503 6504 6505 6506 6507 6508 6509 6510
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  assert(pids != NULL);

  threadInfo *infos = calloc(1, threads * sizeof(threadInfo));
  assert(infos != NULL);

  memset(pids, 0, threads * sizeof(pthread_t));
  memset(infos, 0, threads * sizeof(threadInfo));

H
Hui Li 已提交
6511
  for (int i = 0; i < threads; i++) {
6512 6513
    threadInfo *pThreadInfo = infos + i;
    pThreadInfo->threadID = i;
6514
    tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN);
6515 6516
    pThreadInfo->time_precision = timePrec;
    pThreadInfo->superTblInfo = superTblInfo;
6517

6518 6519
    pThreadInfo->start_time = start_time;
    pThreadInfo->minDelay = UINT64_MAX;
6520

6521
    if ((NULL == superTblInfo) ||
6522 6523
            (superTblInfo->iface != REST_IFACE)) {
      //t_info->taos = taos;
6524
      pThreadInfo->taos = taos_connect(
6525 6526
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
6527
      if (NULL == pThreadInfo->taos) {
6528
        errorPrint(
6529 6530
                "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n",
                __func__, __LINE__,
6531
                taos_errstr(NULL));
6532
        free(infos);
6533 6534
        exit(-1);
      }
6535

6536
#if STMT_IFACE_ENABLED == 1
6537 6538 6539 6540 6541 6542 6543 6544 6545 6546
      if ((g_args.iface == STMT_IFACE)
          || ((superTblInfo) && (superTblInfo->iface == STMT_IFACE))) {

          int columnCount;
          if (superTblInfo) {
              columnCount = superTblInfo->columnCount;
          } else {
              columnCount = g_args.num_of_CPR;
          }

6547 6548
        pThreadInfo->stmt = taos_stmt_init(pThreadInfo->taos);
        if (NULL == pThreadInfo->stmt) {
6549 6550 6551 6552 6553 6554 6555 6556
            errorPrint(
                "%s() LN%d, failed init stmt, reason: %s\n",
                __func__, __LINE__,
                taos_errstr(NULL));
            free(pids);
            free(infos);
            exit(-1);
        }
6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574

        char buffer[3000];
        char *pstr = buffer;
        pstr += sprintf(pstr, "INSERT INTO ? values(?");

        for (int col = 0; col < columnCount; col ++) {
            pstr += sprintf(pstr, ",?");
        }
        pstr += sprintf(pstr, ")");

        int ret = taos_stmt_prepare(pThreadInfo->stmt, buffer, 0);
        if (ret != 0){
            errorPrint("failed to execute taos_stmt_prepare. return 0x%x. reason: %s\n",
                    ret, taos_errstr(NULL));
            free(pids);
            free(infos);
            exit(-1);
        }
6575
      }
6576
#endif
6577
    } else {
6578
      pThreadInfo->taos = NULL;
6579 6580
    }

6581
/*    if ((NULL == superTblInfo)
6582
            || (0 == superTblInfo->multiThreadWriteOneTbl)) {
6583
            */
6584
      pThreadInfo->start_table_from = tableFrom;
6585
      pThreadInfo->ntables = i<b?a+1:a;
6586 6587
      pThreadInfo->end_table_to = i < b ? tableFrom + a : tableFrom + a - 1;
      tableFrom = pThreadInfo->end_table_to + 1;
6588
/*    } else {
6589 6590 6591
      pThreadInfo->start_table_from = 0;
      pThreadInfo->ntables = superTblInfo->childTblCount;
      pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint();
6592
    }
6593
*/
6594
    tsem_init(&(pThreadInfo->lock_sem), 0, 0);
6595
    if (ASYNC_MODE == g_Dbs.asyncMode) {
6596
      pthread_create(pids + i, NULL, asyncWrite, pThreadInfo);
6597
    } else {
6598
      pthread_create(pids + i, NULL, syncWrite, pThreadInfo);
6599
    }
H
Hui Li 已提交
6600
  }
6601

H
Hui Li 已提交
6602 6603 6604 6605
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

6606 6607 6608 6609
  uint64_t totalDelay = 0;
  uint64_t maxDelay = 0;
  uint64_t minDelay = UINT64_MAX;
  uint64_t cntDelay = 1;
6610 6611
  double  avgDelay = 0;

H
Hui Li 已提交
6612
  for (int i = 0; i < threads; i++) {
6613
    threadInfo *pThreadInfo = infos + i;
6614

6615
    tsem_destroy(&(pThreadInfo->lock_sem));
6616

6617
#if STMT_IFACE_ENABLED == 1
6618 6619
    if (pThreadInfo->stmt) {
      taos_stmt_close(pThreadInfo->stmt);
6620
    }
6621
#endif
6622 6623
    tsem_destroy(&(pThreadInfo->lock_sem));
    taos_close(pThreadInfo->taos);
6624

6625
    debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n",
6626
            __func__, __LINE__,
6627 6628
            pThreadInfo->threadID, pThreadInfo->totalInsertRows,
            pThreadInfo->totalAffectedRows);
6629
    if (superTblInfo) {
6630 6631
        superTblInfo->totalAffectedRows += pThreadInfo->totalAffectedRows;
        superTblInfo->totalInsertRows += pThreadInfo->totalInsertRows;
6632
    } else {
6633 6634
        g_args.totalAffectedRows += pThreadInfo->totalAffectedRows;
        g_args.totalInsertRows += pThreadInfo->totalInsertRows;
6635
    }
6636

6637 6638 6639 6640
    totalDelay  += pThreadInfo->totalDelay;
    cntDelay   += pThreadInfo->cntDelay;
    if (pThreadInfo->maxDelay > maxDelay) maxDelay = pThreadInfo->maxDelay;
    if (pThreadInfo->minDelay < minDelay) minDelay = pThreadInfo->minDelay;
H
Hui Li 已提交
6641
  }
6642
  cntDelay -= 1;
H
Hui Li 已提交
6643

6644
  if (cntDelay == 0)    cntDelay = 1;
6645 6646
  avgDelay = (double)totalDelay / cntDelay;

6647 6648
  int64_t end = taosGetTimestampMs();
  int64_t t = end - start;
6649 6650

  if (superTblInfo) {
6651
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
6652 6653 6654 6655
          t / 1000.0, superTblInfo->totalInsertRows,
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
6656 6657 6658 6659

    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
6660 6661 6662 6663
          t / 1000.0, superTblInfo->totalInsertRows,
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
6664
    }
6665
  } else {
6666
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
6667 6668 6669 6670
          t / 1000.0, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          (double)g_args.totalInsertRows / (t / 1000.0));
6671 6672 6673
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
6674 6675 6676 6677
          t * 1000.0, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          (double)g_args.totalInsertRows / (t / 1000.0));
6678
    }
6679
  }
6680

6681
  fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
6682
          avgDelay, maxDelay, minDelay);
6683 6684
  if (g_fpOfInsertResult) {
    fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
6685
          avgDelay, maxDelay, minDelay);
6686
  }
6687

6688 6689
  //taos_close(taos);

H
Hui Li 已提交
6690
  free(pids);
6691
  free(infos);
H
Hui Li 已提交
6692 6693
}

6694
static void *readTable(void *sarg) {
6695
#if 1
6696 6697
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  TAOS *taos = pThreadInfo->taos;
H
hzcheng 已提交
6698
  char command[BUFFER_SIZE] = "\0";
6699 6700 6701
  uint64_t sTime = pThreadInfo->start_time;
  char *tb_prefix = pThreadInfo->tb_prefix;
  FILE *fp = fopen(pThreadInfo->filePath, "a");
H
Hui Li 已提交
6702
  if (NULL == fp) {
6703
    errorPrint( "fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno));
H
Hui Li 已提交
6704 6705
    return NULL;
  }
6706

6707
  int64_t num_of_DPT;
6708 6709
/*  if (pThreadInfo->superTblInfo) {
    num_of_DPT = pThreadInfo->superTblInfo->insertRows; //  nrecords_per_table;
6710
  } else {
6711
  */
6712
      num_of_DPT = g_args.num_of_DPT;
6713
//  }
6714

6715
  int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
S
Shuduo Sang 已提交
6716
  int64_t totalData = num_of_DPT * num_of_tables;
6717
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6718 6719 6720 6721 6722

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
S
Shuduo Sang 已提交
6723
  printf("%"PRId64" records:\n", totalData);
H
hzcheng 已提交
6724 6725
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

6726
  for (int j = 0; j < n; j++) {
H
hzcheng 已提交
6727
    double totalT = 0;
6728
    uint64_t count = 0;
6729 6730
    for (int64_t i = 0; i < num_of_tables; i++) {
      sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64,
6731
              aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
6732

6733
      double t = taosGetTimestampMs();
S
Shuaiqiang Chang 已提交
6734 6735
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
6736

S
Shuaiqiang Chang 已提交
6737
      if (code != 0) {
6738
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6739
        taos_free_result(pSql);
H
hzcheng 已提交
6740
        taos_close(taos);
6741
        fclose(fp);
6742
        return NULL;
H
hzcheng 已提交
6743 6744
      }

6745
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6746 6747 6748
        count++;
      }

6749
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6750 6751
      totalT += t;

S
Shuaiqiang Chang 已提交
6752
      taos_free_result(pSql);
H
hzcheng 已提交
6753 6754
    }

6755
    fprintf(fp, "|%10s  |   %"PRId64"   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
6756
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
6757
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
6758
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000);
H
hzcheng 已提交
6759 6760 6761
  }
  fprintf(fp, "\n");
  fclose(fp);
6762
#endif
H
hzcheng 已提交
6763 6764 6765
  return NULL;
}

6766
static void *readMetric(void *sarg) {
6767
#if 1
6768 6769
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  TAOS *taos = pThreadInfo->taos;
H
hzcheng 已提交
6770
  char command[BUFFER_SIZE] = "\0";
6771
  FILE *fp = fopen(pThreadInfo->filePath, "a");
H
Hui Li 已提交
6772
  if (NULL == fp) {
6773
    printf("fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno));
H
Hui Li 已提交
6774 6775
    return NULL;
  }
6776

6777 6778
  int64_t num_of_DPT = pThreadInfo->superTblInfo->insertRows;
  int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
S
Shuduo Sang 已提交
6779
  int64_t totalData = num_of_DPT * num_of_tables;
6780
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6781 6782 6783 6784 6785

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
S
Shuduo Sang 已提交
6786 6787
  printf("%"PRId64" records:\n", totalData);
  fprintf(fp, "Querying On %"PRId64" records:\n", totalData);
H
hzcheng 已提交
6788 6789

  for (int j = 0; j < n; j++) {
6790
    char condition[COND_BUF_LEN] = "\0";
B
Bomin Zhang 已提交
6791
    char tempS[64] = "\0";
H
hzcheng 已提交
6792

6793
    int64_t m = 10 < num_of_tables ? 10 : num_of_tables;
H
hzcheng 已提交
6794

6795
    for (int64_t i = 1; i <= m; i++) {
H
hzcheng 已提交
6796
      if (i == 1) {
6797
        sprintf(tempS, "t1 = %"PRId64"", i);
H
hzcheng 已提交
6798
      } else {
6799
        sprintf(tempS, " or t1 = %"PRId64" ", i);
H
hzcheng 已提交
6800
      }
6801
      strncat(condition, tempS, COND_BUF_LEN - 1);
H
hzcheng 已提交
6802

L
liu0x54 已提交
6803
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
6804 6805 6806 6807

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

6808
      double t = taosGetTimestampMs();
H
hzcheng 已提交
6809

S
Shuaiqiang Chang 已提交
6810 6811 6812 6813
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
6814
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6815
        taos_free_result(pSql);
H
hzcheng 已提交
6816
        taos_close(taos);
6817
        fclose(fp);
6818
        return NULL;
H
hzcheng 已提交
6819 6820
      }
      int count = 0;
6821
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6822 6823
        count++;
      }
6824
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6825

6826
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
6827 6828
              num_of_tables * num_of_DPT / (t * 1000.0), t);
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0);
H
hzcheng 已提交
6829

S
Shuaiqiang Chang 已提交
6830
      taos_free_result(pSql);
H
hzcheng 已提交
6831 6832 6833 6834
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
6835
#endif
H
hzcheng 已提交
6836 6837 6838
  return NULL;
}

6839 6840 6841 6842 6843 6844 6845
static void prompt()
{
  if (!g_args.answer_yes) {
    printf("         Press enter key to continue or Ctrl-C to stop\n\n");
    (void)getchar();
  }
}
H
Hui Li 已提交
6846

6847
static int insertTestProcess() {
6848

6849 6850 6851
  setupForAnsiEscape();
  int ret = printfInsertMeta();
  resetAfterAnsiEscape();
6852

6853 6854 6855
  if (ret == -1)
    exit(EXIT_FAILURE);

6856
  debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
6857 6858
  g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
  if (NULL == g_fpOfInsertResult) {
6859
    errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
6860 6861
    return -1;
  }
6862

6863 6864
  if (g_fpOfInsertResult)
    printfInsertMetaToFile(g_fpOfInsertResult);
6865

6866
  prompt();
6867

6868 6869 6870
  init_rand_data();

  // create database and super tables
6871
  if(createDatabasesAndStables() != 0) {
6872 6873
    if (g_fpOfInsertResult)
      fclose(g_fpOfInsertResult);
6874 6875
    return -1;
  }
6876 6877

  // pretreatement
6878 6879 6880 6881 6882
  if (prepareSampleData() != 0) {
    if (g_fpOfInsertResult)
      fclose(g_fpOfInsertResult);
    return -1;
  }
6883

6884 6885 6886 6887
  double start;
  double end;

  // create child tables
6888
  start = taosGetTimestampMs();
6889
  createChildTables();
6890
  end = taosGetTimestampMs();
6891

6892
  if (g_totalChildTables > 0) {
6893
    fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6894
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6895 6896
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
6897
            "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6898
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6899
    }
6900
  }
6901

6902
  // create sub threads for inserting data
6903
  //start = taosGetTimestampMs();
6904
  for (int i = 0; i < g_Dbs.dbCount; i++) {
6905 6906
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
S
Shuduo Sang 已提交
6907
        for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6908 6909 6910 6911 6912

          SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];

          if (superTblInfo && (superTblInfo->insertRows > 0)) {
            startMultiThreadInsertData(
6913 6914 6915 6916
              g_Dbs.threadCount,
              g_Dbs.db[i].dbName,
              g_Dbs.db[i].dbCfg.precision,
              superTblInfo);
6917
          }
6918
        }
6919
      }
6920
    } else {
6921
        startMultiThreadInsertData(
6922 6923 6924
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
6925
          NULL);
H
Hui Li 已提交
6926
    }
6927
  }
6928
  //end = taosGetTimestampMs();
6929

6930
  //int64_t    totalInsertRows = 0;
6931
  //int64_t    totalAffectedRows = 0;
6932
  //for (int i = 0; i < g_Dbs.dbCount; i++) {
6933
  //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6934
  //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
6935 6936
  //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
  //}
6937
  //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
6938
  postFreeResource();
6939

6940 6941 6942
  return 0;
}

6943 6944
static void *specifiedTableQuery(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6945

6946
  if (pThreadInfo->taos == NULL) {
6947 6948
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
6949 6950 6951 6952
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
6953 6954
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6955
            pThreadInfo->threadID, taos_errstr(NULL));
6956 6957
      return NULL;
    } else {
6958
      pThreadInfo->taos = taos;
6959 6960 6961
    }
  }

6962
  char sqlStr[TSDB_DB_NAME_LEN + 5];
6963
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6964 6965
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6966 6967 6968 6969
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
    return NULL;
  }
6970

6971 6972
  uint64_t st = 0;
  uint64_t et = 0;
6973

6974
  uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes;
6975

6976 6977 6978
  uint64_t totalQueried = 0;
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
6979

6980
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') {
6981
    sprintf(pThreadInfo->filePath, "%s-%d",
6982 6983 6984 6985
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }

6986
  while(queryTimes --) {
6987
    if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) <
6988
            (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) {
6989
      taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms
6990 6991
    }

6992 6993 6994
    st = taosGetTimestampMs();

    selectAndGetResult(pThreadInfo,
6995
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]);
6996

6997 6998 6999 7000
    et = taosGetTimestampMs();
    printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n",
              taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0);

7001 7002
    totalQueried ++;
    g_queryInfo.specifiedQueryInfo.totalQueried ++;
7003

7004 7005
    uint64_t  currentPrintTime = taosGetTimestampMs();
    uint64_t  endTs = taosGetTimestampMs();
7006
    if (currentPrintTime - lastPrintTime > 30*1000) {
7007
      debugPrint("%s() LN%d, endTs=%"PRIu64"ms, startTs=%"PRIu64"ms\n",
7008
          __func__, __LINE__, endTs, startTs);
7009
      printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.6f\n",
7010 7011
                    pThreadInfo->threadID,
                    totalQueried,
7012
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
7013
      lastPrintTime = currentPrintTime;
7014
    }
H
Hui Li 已提交
7015
  }
7016 7017
  return NULL;
}
H
Hui Li 已提交
7018

7019
static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) {
7020 7021
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
7022 7023
  sprintf(subTblName, "%s.%s",
          g_queryInfo.dbName,
7024
          g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
7025 7026

  //printf("inSql: %s\n", inSql);
7027

7028 7029
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
7030
    return;
H
Hui Li 已提交
7031
  }
7032

7033
  tstrncpy(outSql, inSql, pos - inSql + 1);
7034
  //printf("1: %s\n", outSql);
7035
  strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1);
7036
  //printf("2: %s\n", outSql);
7037
  strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1);
7038
  //printf("3: %s\n", outSql);
H
Hui Li 已提交
7039 7040
}

7041
static void *superTableQuery(void *sarg) {
7042
  char sqlstr[MAX_QUERY_SQL_LENGTH];
7043
  threadInfo *pThreadInfo = (threadInfo *)sarg;
7044

7045
  if (pThreadInfo->taos == NULL) {
7046 7047 7048 7049 7050 7051 7052 7053
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
7054
            pThreadInfo->threadID, taos_errstr(NULL));
7055 7056
      return NULL;
    } else {
7057
      pThreadInfo->taos = taos;
7058 7059 7060
    }
  }

7061 7062
  uint64_t st = 0;
  uint64_t et = (int64_t)g_queryInfo.superQueryInfo.queryInterval;
7063

7064 7065 7066
  uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes;
  uint64_t totalQueried = 0;
  uint64_t  startTs = taosGetTimestampMs();
7067

7068
  uint64_t  lastPrintTime = taosGetTimestampMs();
7069
  while(queryTimes --) {
7070 7071 7072
    if (g_queryInfo.superQueryInfo.queryInterval
            && (et - st) < (int64_t)g_queryInfo.superQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.superQueryInfo.queryInterval - (et - st)); // ms
7073
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to);
7074
    }
H
Hui Li 已提交
7075

7076
    st = taosGetTimestampMs();
7077
    for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) {
7078
      for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
7079
        memset(sqlstr,0,sizeof(sqlstr));
7080
        replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i);
7081
        if (g_queryInfo.superQueryInfo.result[j][0] != '\0') {
7082
          sprintf(pThreadInfo->filePath, "%s-%d",
7083
                  g_queryInfo.superQueryInfo.result[j],
7084
                  pThreadInfo->threadID);
7085
        }
7086
        selectAndGetResult(pThreadInfo, sqlstr);
7087 7088 7089 7090 7091 7092 7093

        totalQueried++;
        g_queryInfo.superQueryInfo.totalQueried ++;

        int64_t  currentPrintTime = taosGetTimestampMs();
        int64_t  endTs = taosGetTimestampMs();
        if (currentPrintTime - lastPrintTime > 30*1000) {
7094
          printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.3f\n",
7095 7096
                    pThreadInfo->threadID,
                    totalQueried,
7097
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
7098
          lastPrintTime = currentPrintTime;
7099
        }
7100
      }
H
Hui Li 已提交
7101
    }
7102
    et = taosGetTimestampMs();
7103
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%"PRIu64" - %"PRIu64"] once queries duration:%.4fs\n\n",
7104
            taosGetSelfPthreadId(),
7105 7106
            pThreadInfo->start_table_from,
            pThreadInfo->end_table_to,
7107
            (double)(et - st)/1000.0);
7108
  }
7109

7110 7111 7112
  return NULL;
}

7113
static int queryTestProcess() {
7114 7115 7116 7117

  setupForAnsiEscape();
  printfQueryMeta();
  resetAfterAnsiEscape();
7118 7119 7120 7121 7122 7123

  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
7124
          g_queryInfo.port);
7125
  if (taos == NULL) {
7126 7127
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
7128 7129 7130
    exit(-1);
  }

7131
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
7132 7133
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
7134 7135 7136
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
7137
  }
7138

7139
  prompt();
7140

7141 7142 7143
  if (g_args.debug_print || g_args.verbose_print) {
    printfQuerySystemInfo(taos);
  }
7144

7145 7146 7147 7148 7149 7150
  if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
    if (convertHostToServAddr(
        g_queryInfo.host, g_queryInfo.port, &g_queryInfo.serv_addr) != 0)
      exit(-1);
  }

7151 7152 7153
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
7154
  int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent;
7155
  uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount;
7156

7157
  uint64_t startTs = taosGetTimestampMs();
7158

7159 7160 7161 7162 7163 7164
  if ((nSqlCount > 0) && (nConcurrent > 0)) {

    pids  = malloc(nConcurrent * nSqlCount * sizeof(pthread_t));
    infos = malloc(nConcurrent * nSqlCount * sizeof(threadInfo));

    if ((NULL == pids) || (NULL == infos)) {
7165
      taos_close(taos);
7166
      ERROR_EXIT("memory allocation failed for create threads\n");
7167
    }
7168

7169 7170 7171
    for (uint64_t i = 0; i < nSqlCount; i++) {
        for (int j = 0; j < nConcurrent; j++) {
            uint64_t seq = i * nConcurrent + j;
7172 7173 7174
            threadInfo *pThreadInfo = infos + seq;
            pThreadInfo->threadID = seq;
            pThreadInfo->querySeq = i;
7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188

            if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {

                char sqlStr[MAX_TB_NAME_SIZE*2];
                sprintf(sqlStr, "use %s", g_queryInfo.dbName);
                if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) {
                    taos_close(taos);
                    free(infos);
                    free(pids);
                    errorPrint( "use database %s failed!\n\n",
                        g_queryInfo.dbName);
                    return -1;
                }
            }
7189

7190
            pThreadInfo->taos = NULL;// TODO: workaround to use separate taos connection;
7191

7192
            pthread_create(pids + seq, NULL, specifiedTableQuery,
7193
                pThreadInfo);
7194
        }
7195
    }
7196
  } else {
7197
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
7198
  }
7199

7200 7201
  taos_close(taos);

7202 7203 7204
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
7205 7206 7207 7208
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
    pidsOfSub  = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t));
    infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo));
7209 7210

    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
7211 7212
      free(infos);
      free(pids);
7213

7214
      ERROR_EXIT("memory allocation failed for create threads\n");
7215
    }
7216

S
Shuduo Sang 已提交
7217
    int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
7218
    int threads = g_queryInfo.superQueryInfo.threadCnt;
7219

S
Shuduo Sang 已提交
7220
    int64_t a = ntables / threads;
7221 7222 7223 7224
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
7225

S
Shuduo Sang 已提交
7226
    int64_t b = 0;
7227 7228 7229
    if (threads != 0) {
      b = ntables % threads;
    }
7230

7231
    uint64_t tableFrom = 0;
7232
    for (int i = 0; i < threads; i++) {
7233 7234
      threadInfo *pThreadInfo = infosOfSub + i;
      pThreadInfo->threadID = i;
7235

7236
      pThreadInfo->start_table_from = tableFrom;
7237
      pThreadInfo->ntables = i<b?a+1:a;
7238 7239
      pThreadInfo->end_table_to = i < b ? tableFrom + a : tableFrom + a - 1;
      tableFrom = pThreadInfo->end_table_to + 1;
7240 7241
      pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection;
      pthread_create(pidsOfSub + i, NULL, superTableQuery, pThreadInfo);
7242 7243
    }

7244
    g_queryInfo.superQueryInfo.threadCnt = threads;
7245
  } else {
7246
    g_queryInfo.superQueryInfo.threadCnt = 0;
7247
  }
7248

7249 7250 7251 7252 7253 7254
  if ((nSqlCount > 0) && (nConcurrent > 0)) {
    for (int i = 0; i < nConcurrent; i++) {
      for (int j = 0; j < nSqlCount; j++) {
        pthread_join(pids[i * nSqlCount + j], NULL);
      }
    }
7255
  }
S
Shuaiqiang Chang 已提交
7256

7257
  tmfree((char*)pids);
7258
  tmfree((char*)infos);
7259

7260
  for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
7261
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
7262
  }
H
Hui Li 已提交
7263

7264
  tmfree((char*)pidsOfSub);
7265
  tmfree((char*)infosOfSub);
7266

7267
//  taos_close(taos);// TODO: workaround to use separate taos connection;
7268
  uint64_t endTs = taosGetTimestampMs();
7269

7270
  uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried +
7271 7272
    g_queryInfo.superQueryInfo.totalQueried;

7273
  fprintf(stderr, "==== completed total queries: %"PRIu64", the QPS of all threads: %10.3f====\n",
7274
          totalQueried,
7275
          (double)(totalQueried/((endTs-startTs)/1000.0)));
7276 7277 7278
  return 0;
}

7279 7280 7281 7282 7283 7284 7285 7286 7287
static void stable_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
  if (res == NULL || taos_errno(res) != 0) {
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
    return;
  }

  if (param)
7288
    fetchResult(res, (threadInfo *)param);
7289 7290 7291 7292 7293
  // tao_unscribe() will free result.
}

static void specified_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
7294
  if (res == NULL || taos_errno(res) != 0) {
7295 7296
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
7297 7298
    return;
  }
7299

7300
  if (param)
7301
    fetchResult(res, (threadInfo *)param);
7302
  // tao_unscribe() will free result.
H
hzcheng 已提交
7303 7304
}

7305
static TAOS_SUB* subscribeImpl(
7306 7307 7308 7309
        QUERY_CLASS class,
        threadInfo *pThreadInfo,
        char *sql, char* topic, bool restart, uint64_t interval)
{
7310
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
7311

7312 7313 7314 7315
  if ((SPECIFIED_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) {
    tsub = taos_subscribe(
            pThreadInfo->taos,
7316
            restart,
7317
            topic, sql, specified_sub_callback, (void*)pThreadInfo,
7318
            g_queryInfo.specifiedQueryInfo.subscribeInterval);
7319 7320 7321 7322 7323 7324 7325
  } else if ((STABLE_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) {
    tsub = taos_subscribe(
            pThreadInfo->taos,
            restart,
            topic, sql, stable_sub_callback, (void*)pThreadInfo,
            g_queryInfo.superQueryInfo.subscribeInterval);
7326
  } else {
7327 7328
    tsub = taos_subscribe(
            pThreadInfo->taos,
7329
            restart,
7330
            topic, sql, NULL, NULL, interval);
7331
  }
7332

7333
  if (tsub == NULL) {
7334 7335
      errorPrint("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
      return NULL;
7336
  }
7337

7338 7339
  return tsub;
}
H
hzcheng 已提交
7340

7341
static void *superSubscribe(void *sarg) {
7342 7343 7344 7345 7346 7347 7348 7349 7350 7351
    threadInfo *pThreadInfo = (threadInfo *)sarg;
    char subSqlstr[MAX_QUERY_SQL_LENGTH];
    TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT] = {0};
    uint64_t tsubSeq;

    if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
        errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n",
                pThreadInfo->ntables, MAX_QUERY_SQL_COUNT);
        exit(-1);
    }
7352

7353
    if (pThreadInfo->taos == NULL) {
7354 7355 7356 7357 7358 7359 7360 7361 7362 7363
        pThreadInfo->taos = taos_connect(g_queryInfo.host,
                g_queryInfo.user,
                g_queryInfo.password,
                g_queryInfo.dbName,
                g_queryInfo.port);
        if (pThreadInfo->taos == NULL) {
            errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
                    pThreadInfo->threadID, taos_errstr(NULL));
            return NULL;
        }
7364 7365
    }

7366 7367 7368 7369 7370
    char sqlStr[MAX_TB_NAME_SIZE*2];
    sprintf(sqlStr, "use %s", g_queryInfo.dbName);
    if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
        taos_close(pThreadInfo->taos);
        errorPrint( "use database %s failed!\n\n",
7371
                g_queryInfo.dbName);
7372 7373
        return NULL;
    }
7374

7375 7376 7377 7378 7379 7380 7381 7382 7383 7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394
    char topic[32] = {0};
    for (uint64_t i = pThreadInfo->start_table_from;
            i <= pThreadInfo->end_table_to; i++) {
        tsubSeq = i - pThreadInfo->start_table_from;
        verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n",
                __func__, __LINE__,
                pThreadInfo->threadID,
                pThreadInfo->start_table_from,
                pThreadInfo->end_table_to, i);
        sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"",
                i, pThreadInfo->querySeq);
        memset(subSqlstr, 0, sizeof(subSqlstr));
        replaceChildTblName(
                g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq],
                subSqlstr, i);
        if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
            sprintf(pThreadInfo->filePath, "%s-%d",
                    g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                    pThreadInfo->threadID);
        }
7395

7396
        verbosePrint("%s() LN%d, [%d] subSqlstr: %s\n",
7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407
                __func__, __LINE__, pThreadInfo->threadID, subSqlstr);
        tsub[tsubSeq] = subscribeImpl(
                STABLE_CLASS,
                pThreadInfo, subSqlstr, topic,
                g_queryInfo.superQueryInfo.subscribeRestart,
                g_queryInfo.superQueryInfo.subscribeInterval);
        if (NULL == tsub[tsubSeq]) {
            taos_close(pThreadInfo->taos);
            return NULL;
        }
    }
7408

7409 7410 7411 7412 7413 7414
    // start loop to consume result
    int consumed[MAX_QUERY_SQL_COUNT];
    for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) {
        consumed[i] = 0;
    }
    TAOS_RES* res = NULL;
7415

7416
    uint64_t st = 0, et = 0;
7417

7418 7419 7420 7421
    while ((g_queryInfo.superQueryInfo.endAfterConsume == -1)
            || (g_queryInfo.superQueryInfo.endAfterConsume >
                consumed[pThreadInfo->end_table_to
                - pThreadInfo->start_table_from])) {
7422

7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439 7440 7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474 7475 7476 7477
        verbosePrint("super endAfterConsume: %d, consumed: %d\n",
                g_queryInfo.superQueryInfo.endAfterConsume,
                consumed[pThreadInfo->end_table_to
                - pThreadInfo->start_table_from]);
        for (uint64_t i = pThreadInfo->start_table_from;
                i <= pThreadInfo->end_table_to; i++) {
            tsubSeq = i - pThreadInfo->start_table_from;
            if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
                continue;
            }

            st = taosGetTimestampMs();
            performancePrint("st: %"PRIu64" et: %"PRIu64" st-et: %"PRIu64"\n", st, et, (st - et));
            res = taos_consume(tsub[tsubSeq]);
            et = taosGetTimestampMs();
            performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st));

            if (res) {
                if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
                    sprintf(pThreadInfo->filePath, "%s-%d",
                            g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                            pThreadInfo->threadID);
                    fetchResult(res, pThreadInfo);
                }
                consumed[tsubSeq] ++;

                if ((g_queryInfo.superQueryInfo.resubAfterConsume != -1)
                        && (consumed[tsubSeq] >=
                            g_queryInfo.superQueryInfo.resubAfterConsume)) {
                    verbosePrint("%s() LN%d, keepProgress:%d, resub super table query: %"PRIu64"\n",
                            __func__, __LINE__,
                            g_queryInfo.superQueryInfo.subscribeKeepProgress,
                            pThreadInfo->querySeq);
                    taos_unsubscribe(tsub[tsubSeq],
                            g_queryInfo.superQueryInfo.subscribeKeepProgress);
                    consumed[tsubSeq]= 0;
                    tsub[tsubSeq] = subscribeImpl(
                            STABLE_CLASS,
                            pThreadInfo, subSqlstr, topic,
                            g_queryInfo.superQueryInfo.subscribeRestart,
                            g_queryInfo.superQueryInfo.subscribeInterval
                            );
                    if (NULL == tsub[tsubSeq]) {
                        taos_close(pThreadInfo->taos);
                        return NULL;
                    }
                }
            }
        }
    }
    verbosePrint("%s() LN%d, super endAfterConsume: %d, consumed: %d\n",
            __func__, __LINE__,
            g_queryInfo.superQueryInfo.endAfterConsume,
            consumed[pThreadInfo->end_table_to - pThreadInfo->start_table_from]);
    taos_free_result(res);
7478

7479 7480
    for (uint64_t i = pThreadInfo->start_table_from;
            i <= pThreadInfo->end_table_to; i++) {
7481 7482
        tsubSeq = i - pThreadInfo->start_table_from;
        taos_unsubscribe(tsub[tsubSeq], 0);
H
hzcheng 已提交
7483
    }
7484

7485 7486
    taos_close(pThreadInfo->taos);
    return NULL;
H
hzcheng 已提交
7487 7488
}

7489 7490
static void *specifiedSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
7491
//  TAOS_SUB*  tsub = NULL;
7492

7493
  if (pThreadInfo->taos == NULL) {
7494
    pThreadInfo->taos = taos_connect(g_queryInfo.host,
7495 7496 7497 7498
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
7499
    if (pThreadInfo->taos == NULL) {
7500
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
7501
            pThreadInfo->threadID, taos_errstr(NULL));
7502 7503 7504 7505
      return NULL;
    }
  }

7506 7507
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
7508 7509
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
7510 7511
    return NULL;
  }
7512

7513 7514 7515 7516
  sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
          "taosdemo-subscribe-%"PRIu64"-%d",
          pThreadInfo->querySeq,
          pThreadInfo->threadID);
7517
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') {
7518
      sprintf(pThreadInfo->filePath, "%s-%d",
7519 7520 7521
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }
7522 7523
  g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl(
          SPECIFIED_CLASS, pThreadInfo,
7524
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
7525
          g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
7526
          g_queryInfo.specifiedQueryInfo.subscribeRestart,
7527
          g_queryInfo.specifiedQueryInfo.subscribeInterval);
7528
  if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) {
7529 7530
      taos_close(pThreadInfo->taos);
      return NULL;
S
Shengliang Guan 已提交
7531
  }
7532

7533
  // start loop to consume result
7534

7535
  g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0;
7536 7537 7538 7539
  while((g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq] == -1)
          || (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] <
              g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq])) {

7540 7541 7542 7543 7544
      printf("consumed[%d]: %d, endAfterConsum[%"PRId64"]: %d\n",
              pThreadInfo->threadID,
              g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID],
              pThreadInfo->querySeq,
              g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq]);
7545
      if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) {
7546
          continue;
7547
      }
7548

7549 7550 7551
      g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] = taos_consume(
              g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]);
      if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) {
7552 7553
          if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0]
                  != 0) {
7554
              sprintf(pThreadInfo->filePath, "%s-%d",
7555 7556 7557
                      g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
          }
7558 7559 7560
          fetchResult(
                  g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID],
                  pThreadInfo);
7561

7562
          g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] ++;
7563
          if ((g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq] != -1)
7564
                && (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] >=
7565 7566 7567 7568
                    g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
              printf("keepProgress:%d, resub specified query: %"PRIu64"\n",
                    g_queryInfo.specifiedQueryInfo.subscribeKeepProgress,
                    pThreadInfo->querySeq);
7569 7570
              g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0;
              taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID],
7571
                      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
7572 7573 7574 7575 7576 7577 7578 7579
              g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] =
                  subscribeImpl(
                          SPECIFIED_CLASS,
                          pThreadInfo,
                          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
                          g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
                          g_queryInfo.specifiedQueryInfo.subscribeRestart,
                          g_queryInfo.specifiedQueryInfo.subscribeInterval);
7580
              if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) {
7581 7582
                  taos_close(pThreadInfo->taos);
                  return NULL;
7583 7584
              }
          }
7585 7586
      }
  }
7587
  taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]);
7588
  taos_close(pThreadInfo->taos);
7589

H
hzcheng 已提交
7590 7591 7592
  return NULL;
}

7593
static int subscribeTestProcess() {
7594
  setupForAnsiEscape();
7595
  printfQueryMeta();
7596
  resetAfterAnsiEscape();
7597

7598
  prompt();
7599

7600 7601 7602 7603 7604
  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
7605
          g_queryInfo.port);
7606
  if (taos == NULL) {
7607 7608
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
7609 7610 7611
    exit(-1);
  }

7612
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
7613 7614
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
7615 7616 7617
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
7618 7619
  }

7620 7621
  taos_close(taos); // TODO: workaround to use separate taos connection;

7622 7623
  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
7624 7625 7626 7627 7628

  pthread_t  *pidsOfStable  = NULL;
  threadInfo *infosOfStable = NULL;

  //==== create threads for query for specified table
S
Shengliang Guan 已提交
7629
  if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) {
7630
    debugPrint("%s() LN%d, sepcified query sqlCount %d.\n",
7631
              __func__, __LINE__,
S
Shengliang Guan 已提交
7632 7633 7634
              g_queryInfo.specifiedQueryInfo.sqlCount);
  } else {
    if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) {
7635
        errorPrint("%s() LN%d, sepcified query sqlCount %d.\n",
S
Shengliang Guan 已提交
7636 7637 7638 7639
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount);
        exit(-1);
    }
7640

7641 7642 7643 7644 7645 7646 7647 7648
    pids  = malloc(
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(pthread_t));
    infos = malloc(
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(threadInfo));
S
Shengliang Guan 已提交
7649 7650 7651 7652
    if ((NULL == pids) || (NULL == infos)) {
        errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
        exit(-1);
    }
7653

7654 7655 7656
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
            uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
7657 7658 7659 7660 7661
            threadInfo *pThreadInfo = infos + seq;
            pThreadInfo->threadID = seq;
            pThreadInfo->querySeq = i;
            pThreadInfo->taos = NULL;  // TODO: workaround to use separate taos connection;
            pthread_create(pids + seq, NULL, specifiedSubscribe, pThreadInfo);
7662
        }
S
Shengliang Guan 已提交
7663
    }
H
hzcheng 已提交
7664
  }
7665

7666 7667
  //==== create threads for super table query
  if (g_queryInfo.superQueryInfo.sqlCount <= 0) {
7668
    debugPrint("%s() LN%d, super table query sqlCount %d.\n",
7669 7670 7671 7672
              __func__, __LINE__,
              g_queryInfo.superQueryInfo.sqlCount);
  } else {
    if ((g_queryInfo.superQueryInfo.sqlCount > 0)
7673
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
7674 7675 7676
        pidsOfStable  = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
7677
            sizeof(pthread_t));
7678 7679 7680
        infosOfStable = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
7681
            sizeof(threadInfo));
7682 7683
        if ((NULL == pidsOfStable) || (NULL == infosOfStable)) {
            errorPrint("%s() LN%d, malloc failed for create threads\n",
7684
              __func__, __LINE__);
7685 7686 7687
            // taos_close(taos);
            exit(-1);
        }
7688

7689 7690
        int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
        int threads = g_queryInfo.superQueryInfo.threadCnt;
7691

7692 7693 7694 7695 7696
        int64_t a = ntables / threads;
        if (a < 1) {
            threads = ntables;
            a = 1;
        }
7697

7698 7699 7700 7701
        int64_t b = 0;
        if (threads != 0) {
            b = ntables % threads;
        }
7702

7703
        for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
7704
            uint64_t tableFrom = 0;
7705 7706
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
7707 7708 7709 7710
                threadInfo *pThreadInfo = infosOfStable + seq;
                pThreadInfo->threadID = seq;
                pThreadInfo->querySeq = i;

7711
                pThreadInfo->start_table_from = tableFrom;
7712
                pThreadInfo->ntables = j<b?a+1:a;
7713 7714
                pThreadInfo->end_table_to = j<b?tableFrom+a:tableFrom+a-1;
                tableFrom = pThreadInfo->end_table_to + 1;
7715
                pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection;
7716
                pthread_create(pidsOfStable + seq,
7717
                        NULL, superSubscribe, pThreadInfo);
7718 7719
            }
        }
7720

7721
        g_queryInfo.superQueryInfo.threadCnt = threads;
7722

7723 7724 7725 7726 7727 7728
        for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                pthread_join(pidsOfStable[seq], NULL);
            }
        }
7729
    }
H
hzcheng 已提交
7730
  }
7731

7732 7733 7734 7735 7736
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
    for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
        uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
        pthread_join(pids[seq], NULL);
    }
7737
  }
H
hzcheng 已提交
7738

7739
  tmfree((char*)pids);
7740
  tmfree((char*)infos);
H
hzcheng 已提交
7741

7742 7743
  tmfree((char*)pidsOfStable);
  tmfree((char*)infosOfStable);
7744
//   taos_close(taos);
7745
  return 0;
H
hzcheng 已提交
7746 7747
}

7748
static void initOfInsertMeta() {
7749
  memset(&g_Dbs, 0, sizeof(SDbs));
7750

7751
  // set default values
7752
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7753
  g_Dbs.port = 6030;
7754 7755
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
7756
  g_Dbs.threadCount = 2;
7757 7758

  g_Dbs.use_metric = g_args.use_metric;
L
Liu Tao 已提交
7759 7760
}

7761
static void initOfQueryMeta() {
7762
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
7763

7764
  // set default values
7765
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7766
  g_queryInfo.port = 6030;
7767 7768
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
L
Liu Tao 已提交
7769 7770
}

7771
static void setParaFromArg(){
7772
  if (g_args.host) {
7773
    tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE);
7774
  } else {
7775
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7776
  }
L
Liu Tao 已提交
7777

7778
  if (g_args.user) {
7779
    tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE);
7780
  }
7781 7782

  if (g_args.password) {
7783
    tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE);
7784 7785
  }

7786 7787
  if (g_args.port) {
    g_Dbs.port = g_args.port;
7788
  }
L
Liu Tao 已提交
7789

7790 7791 7792
  g_Dbs.threadCount = g_args.num_of_threads;
  g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;

7793
  g_Dbs.dbCount = 1;
7794
  g_Dbs.db[0].drop = true;
7795

7796
  tstrncpy(g_Dbs.db[0].dbName, g_args.database, TSDB_DB_NAME_LEN);
7797
  g_Dbs.db[0].dbCfg.replica = g_args.replica;
7798
  tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", 8);
L
Liu Tao 已提交
7799

7800
  tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
7801 7802 7803 7804 7805

  g_Dbs.use_metric = g_args.use_metric;
  g_Dbs.insert_only = g_args.insert_only;

  g_Dbs.do_aggreFunc = true;
L
Liu Tao 已提交
7806

7807 7808
  char dataString[STRING_LEN];
  char **data_type = g_args.datatype;
7809

7810
  memset(dataString, 0, STRING_LEN);
L
Liu Tao 已提交
7811

7812 7813
  if (strcasecmp(data_type[0], "BINARY") == 0
          || strcasecmp(data_type[0], "BOOL") == 0
7814
          || strcasecmp(data_type[0], "NCHAR") == 0 ) {
7815
    g_Dbs.do_aggreFunc = false;
L
Liu Tao 已提交
7816
  }
H
hzcheng 已提交
7817

7818 7819
  if (g_args.use_metric) {
    g_Dbs.db[0].superTblCount = 1;
7820
    tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", TSDB_TABLE_NAME_LEN);
7821 7822
    g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
    g_Dbs.threadCount = g_args.num_of_threads;
7823
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
7824
    g_Dbs.asyncMode = g_args.async_mode;
7825

7826 7827 7828 7829
    g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
    g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
    g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
7830
    tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix,
7831
            g_args.tb_prefix, TSDB_TABLE_NAME_LEN - 20);
7832
    tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
7833
    g_Dbs.db[0].superTbls[0].iface = g_args.iface;
7834
    tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp,
7835
            "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
7836
    g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
7837

7838
    g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
7839
    g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len;
7840

7841 7842 7843 7844 7845
    g_Dbs.db[0].superTbls[0].columnCount = 0;
    for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
      if (data_type[i] == NULL) {
        break;
      }
7846

7847
      tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
7848
              data_type[i], strlen(data_type[i]) + 1);
7849
      g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;
7850 7851
      g_Dbs.db[0].superTbls[0].columnCount++;
    }
7852

7853 7854 7855
    if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
      g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
    } else {
7856 7857 7858
      for (int i = g_Dbs.db[0].superTbls[0].columnCount;
              i < g_args.num_of_CPR; i++) {
        tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
7859
                "INT", strlen("INT") + 1);
7860
        g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;
7861 7862 7863
        g_Dbs.db[0].superTbls[0].columnCount++;
      }
    }
7864

7865
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType,
7866
            "INT", strlen("INT") + 1);
7867
    g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;
7868

7869
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType,
7870
            "BINARY", strlen("BINARY") + 1);
7871 7872
    g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;
    g_Dbs.db[0].superTbls[0].tagCount = 2;
7873
  } else {
7874 7875
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
    g_Dbs.db[0].superTbls[0].tagCount = 0;
H
hzcheng 已提交
7876
  }
7877 7878 7879 7880 7881 7882
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
7883

7884 7885 7886
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
7887 7888 7889
    exit(-1);
  }

7890 7891 7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910 7911 7912 7913
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

7914
static void querySqlFile(TAOS* taos, char* sqlFile)
7915 7916 7917 7918 7919 7920
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
7921

7922 7923 7924 7925 7926 7927
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

7928
  double t = taosGetTimestampMs();
7929

7930
  while((read_len = tgetline(&line, &line_len, fp)) != -1) {
7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
7943
    }
7944 7945

    memcpy(cmd + cmd_len, line, read_len);
7946 7947 7948
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) {
        errorPrint("%s() LN%d, queryDbExec %s failed!\n",
               __func__, __LINE__, cmd);
7949 7950 7951 7952 7953
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
7954 7955
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
7956 7957
  }

7958
  t = taosGetTimestampMs() - t;
7959
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
7960

7961 7962 7963 7964
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
7965 7966
}

7967
static void testMetaFile() {
7968
    if (INSERT_TEST == g_args.test_mode) {
7969 7970
      if (g_Dbs.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
7971

7972
      insertTestProcess();
7973

7974
    } else if (QUERY_TEST == g_args.test_mode) {
7975 7976
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7977

7978
      queryTestProcess();
7979

7980
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
7981 7982
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7983

7984
      subscribeTestProcess();
7985

7986 7987 7988
    }  else {
      ;
    }
7989
}
7990

7991
static void queryResult() {
7992
  // query data
7993

7994
  pthread_t read_id;
7995 7996 7997 7998
  threadInfo *pThreadInfo = malloc(sizeof(threadInfo));
  assert(pThreadInfo);
  pThreadInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
  pThreadInfo->start_table_from = 0;
7999

8000
  //pThreadInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
8001
  if (g_args.use_metric) {
8002 8003 8004 8005
    pThreadInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
    pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
    pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
    tstrncpy(pThreadInfo->tb_prefix,
8006
          g_Dbs.db[0].superTbls[0].childTblPrefix, TSDB_TABLE_NAME_LEN - 20);
8007
  } else {
8008 8009
    pThreadInfo->ntables = g_args.num_of_tables;
    pThreadInfo->end_table_to = g_args.num_of_tables -1;
8010
    tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, TSDB_TABLE_NAME_LEN);
8011 8012
  }

8013
  pThreadInfo->taos = taos_connect(
8014 8015 8016 8017 8018
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
          g_Dbs.port);
8019
  if (pThreadInfo->taos == NULL) {
8020 8021
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
8022
    free(pThreadInfo);
8023 8024 8025
    exit(-1);
  }

8026
  tstrncpy(pThreadInfo->filePath, g_Dbs.resultFile, MAX_FILE_NAME_LEN);
8027 8028

  if (!g_Dbs.use_metric) {
8029
    pthread_create(&read_id, NULL, readTable, pThreadInfo);
8030
  } else {
8031
    pthread_create(&read_id, NULL, readMetric, pThreadInfo);
8032 8033
  }
  pthread_join(read_id, NULL);
8034 8035
  taos_close(pThreadInfo->taos);
  free(pThreadInfo);
8036 8037
}

8038 8039
static void testCmdLine() {

8040 8041 8042 8043 8044 8045 8046 8047 8048 8049
  if (strlen(configDir)) {
    wordexp_t full_path;
    if (wordexp(configDir, &full_path, 0) != 0) {
      errorPrint( "Invalid path %s\n", configDir);
      return;
    }
    taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
    wordfree(&full_path);
  }

8050 8051
  g_args.test_mode = INSERT_TEST;
  insertTestProcess();
8052

8053
  if (false == g_Dbs.insert_only)
8054
    queryResult();
8055 8056
}

8057 8058 8059
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

8060
  debugPrint("meta file: %s\n", g_args.metaFile);
8061 8062 8063

  if (g_args.metaFile) {
    initOfInsertMeta();
8064
    initOfQueryMeta();
8065 8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076 8077

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
8078 8079 8080 8081
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
8082
          g_Dbs.port);
8083
      querySqlFile(qtaos, g_args.sqlFile);
8084 8085 8086 8087 8088
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
8089 8090 8091

    if (g_dupstr)
        free(g_dupstr);
H
hzcheng 已提交
8092
  }
8093 8094

  return 0;
H
hzcheng 已提交
8095
}
8096