taosdemo.c 290.4 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

21
#include <stdint.h>
22
#include <taos.h>
S
slguan 已提交
23
#define _GNU_SOURCE
24 25
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
42
#else
S
TD-1057  
Shengliang Guan 已提交
43 44
  #include <regex.h>
  #include <stdio.h>
45
#endif
H
hzcheng 已提交
46

47
#include <assert.h>
48
#include <stdlib.h>
49 50
#include "cJSON.h"

51
#include "os.h"
H
hzcheng 已提交
52
#include "taos.h"
53
#include "taoserror.h"
H
Hui Li 已提交
54
#include "tutil.h"
S
slguan 已提交
55

56
#define STMT_IFACE_ENABLED  1
57
#define NANO_SECOND_ENABLED  1
58

59 60 61
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
62 63
extern char configDir[];

64 65 66 67
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

68 69
#define STR_INSERT_INTO     "INSERT INTO "

70 71
#define MAX_RECORDS_PER_REQ     32766

72
#define HEAD_BUFF_LEN       TSDB_MAX_COLUMNS*24  // 16*MAX_COLUMNS + (192+32)*2 + insert into ..
73

74 75
#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
76
#define COND_BUF_LEN       (BUFFER_SIZE - 30)
77 78 79
#define MAX_USERNAME_SIZE  64
#define MAX_PASSWORD_SIZE  64
#define MAX_HOSTNAME_SIZE  64
80
#define MAX_TB_NAME_SIZE   64
81
#define MAX_DATA_SIZE      (16*TSDB_MAX_COLUMNS)+20     // max record len: 16*MAX_COLUMNS, timestamp string and ,('') need extra space
82 83 84
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
85
#define MAX_FILE_NAME_LEN  256              // max file name length on linux is 255.
86

87 88
#define MAX_SAMPLES_ONCE_FROM_FILE   10000
#define MAX_NUM_COLUMNS        (TSDB_MAX_COLUMNS - 1)      // exclude first column timestamp
89

90 91
#define MAX_DB_COUNT            8
#define MAX_SUPER_TABLE_COUNT   200
92

93 94
#define MAX_QUERY_SQL_COUNT     100
#define MAX_QUERY_SQL_LENGTH    1024
95

96 97
#define MAX_DATABASE_COUNT      256
#define INPUT_BUF_LEN           256
98

99
#define DEFAULT_TIMESTAMP_STEP  1
100

101

102 103 104 105 106 107 108
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};

109
typedef enum CREATE_SUB_TALBE_MOD_EN {
110 111 112
    PRE_CREATE_SUBTBL,
    AUTO_CREATE_SUBTBL,
    NO_CREATE_SUBTBL
113
} CREATE_SUB_TALBE_MOD_EN;
114

115
typedef enum TALBE_EXISTS_EN {
116 117 118
    TBL_NO_EXISTS,
    TBL_ALREADY_EXISTS,
    TBL_EXISTS_BUTT
119 120
} TALBE_EXISTS_EN;

121
enum enumSYNC_MODE {
122 123 124
    SYNC_MODE,
    ASYNC_MODE,
    MODE_BUT
125
};
126

S
Shuduo Sang 已提交
127
enum enum_TAOS_INTERFACE {
128 129 130
    TAOSC_IFACE,
    REST_IFACE,
    STMT_IFACE,
131 132 133
    INTERFACE_BUT
};

134 135 136 137 138 139
typedef enum enumQUERY_CLASS {
    SPECIFIED_CLASS,
    STABLE_CLASS,
    CLASS_BUT
} QUERY_CLASS;

S
Shuduo Sang 已提交
140
typedef enum enum_PROGRESSIVE_OR_INTERLACE {
141 142 143
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
144
} PROG_OR_INTERLACE_MODE;
145

146
typedef enum enumQUERY_TYPE {
147 148 149
    NO_INSERT_TYPE,
    INSERT_TYPE,
    QUERY_TYPE_BUT
150
} QUERY_TYPE;
151 152

enum _show_db_index {
153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
    TSDB_SHOW_DB_NAME_INDEX,
    TSDB_SHOW_DB_CREATED_TIME_INDEX,
    TSDB_SHOW_DB_NTABLES_INDEX,
    TSDB_SHOW_DB_VGROUPS_INDEX,
    TSDB_SHOW_DB_REPLICA_INDEX,
    TSDB_SHOW_DB_QUORUM_INDEX,
    TSDB_SHOW_DB_DAYS_INDEX,
    TSDB_SHOW_DB_KEEP_INDEX,
    TSDB_SHOW_DB_CACHE_INDEX,
    TSDB_SHOW_DB_BLOCKS_INDEX,
    TSDB_SHOW_DB_MINROWS_INDEX,
    TSDB_SHOW_DB_MAXROWS_INDEX,
    TSDB_SHOW_DB_WALLEVEL_INDEX,
    TSDB_SHOW_DB_FSYNC_INDEX,
    TSDB_SHOW_DB_COMP_INDEX,
    TSDB_SHOW_DB_CACHELAST_INDEX,
    TSDB_SHOW_DB_PRECISION_INDEX,
    TSDB_SHOW_DB_UPDATE_INDEX,
    TSDB_SHOW_DB_STATUS_INDEX,
    TSDB_MAX_SHOW_DB
173 174 175 176
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
177 178 179 180 181 182 183 184
    TSDB_SHOW_STABLES_NAME_INDEX,
    TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
    TSDB_SHOW_STABLES_COLUMNS_INDEX,
    TSDB_SHOW_STABLES_METRIC_INDEX,
    TSDB_SHOW_STABLES_UID_INDEX,
    TSDB_SHOW_STABLES_TID_INDEX,
    TSDB_SHOW_STABLES_VGID_INDEX,
    TSDB_MAX_SHOW_STABLES
185 186
};

187
enum _describe_table_index {
188 189 190 191 192
    TSDB_DESCRIBE_METRIC_FIELD_INDEX,
    TSDB_DESCRIBE_METRIC_TYPE_INDEX,
    TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
    TSDB_DESCRIBE_METRIC_NOTE_INDEX,
    TSDB_MAX_DESCRIBE_METRIC
193 194
};

H
hzcheng 已提交
195
/* Used by main to communicate with parse_opt. */
196 197
static char *g_dupstr = NULL;

198
typedef struct SArguments_S {
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
    char *   metaFile;
    uint32_t test_mode;
    char *   host;
    uint16_t port;
    uint16_t iface;
    char *   user;
    char *   password;
    char *   database;
    int      replica;
    char *   tb_prefix;
    char *   sqlFile;
    bool     use_metric;
    bool     drop_database;
    bool     insert_only;
    bool     answer_yes;
    bool     debug_print;
    bool     verbose_print;
    bool     performance_print;
    char *   output_file;
    bool     async_mode;
219
    char *   datatype[MAX_NUM_COLUMNS + 1];
220 221 222 223 224 225 226 227 228 229 230 231
    uint32_t len_of_binary;
    uint32_t num_of_CPR;
    uint32_t num_of_threads;
    uint64_t insert_interval;
    int64_t  query_times;
    uint32_t interlace_rows;
    uint32_t num_of_RPR;                  // num_of_records_per_req
    uint64_t max_sql_len;
    int64_t  num_of_tables;
    int64_t  num_of_DPT;
    int      abort;
    uint32_t disorderRatio;               // 0: no disorder, >0: x%
232
    int      disorderRange;               // ms, us or ns. accordig to database precision
233 234 235 236 237
    uint32_t method_of_delete;
    char **  arg_list;
    uint64_t totalInsertRows;
    uint64_t totalAffectedRows;
    bool     demo_mode;                  // use default column name and semi-random data
238 239 240
} SArguments;

typedef struct SColumn_S {
241 242 243 244
    char      field[TSDB_COL_NAME_LEN];
    char      dataType[16];
    uint32_t  dataLen;
    char      note[128];
245 246 247
} StrColumn;

typedef struct SSuperTable_S {
248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
    char         sTblName[TSDB_TABLE_NAME_LEN];
    char         dataSource[MAX_TB_NAME_SIZE];  // rand_gen or sample
    char         childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq
    char         insertMode[MAX_TB_NAME_SIZE];    // taosc, rest
    uint16_t     childTblExists;
    int64_t      childTblCount;
    uint64_t     batchCreateTableNum;     // 0: no batch,  > 0: batch table number in one sql
    uint8_t      autoCreateTable;         // 0: create sub table, 1: auto create sub table
    uint16_t     iface;                   // 0: taosc, 1: rest, 2: stmt
    int64_t      childTblLimit;
    uint64_t     childTblOffset;

    //  int          multiThreadWriteOneTbl;  // 0: no, 1: yes
    uint32_t     interlaceRows;           //
    int          disorderRatio;           // 0: no disorder, >0: x%
    int          disorderRange;           // ms, us or ns. according to database precision
    uint64_t     maxSqlLen;               //

    uint64_t     insertInterval;          // insert interval, will override global insert interval
    int64_t      insertRows;
    int64_t      timeStampStep;
    char         startTimestamp[MAX_TB_NAME_SIZE];
    char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
    char         sampleFile[MAX_FILE_NAME_LEN];
    char         tagsFile[MAX_FILE_NAME_LEN];

    uint32_t     columnCount;
    StrColumn    columns[TSDB_MAX_COLUMNS];
    uint32_t     tagCount;
    StrColumn    tags[TSDB_MAX_TAGS];

    char*        childTblName;
    char*        colsOfCreateChildTable;
    uint64_t     lenOfOneRow;
    uint64_t     lenOfTagOfOneRow;

    char*        sampleDataBuf;
    //int          sampleRowCount;
    //int          sampleUsePos;

    uint32_t     tagSource;    // 0: rand, 1: tag sample
    char*        tagDataBuf;
    uint32_t     tagSampleCount;
    uint32_t     tagUsePos;

    // statistics
    uint64_t     totalInsertRows;
    uint64_t     totalAffectedRows;
296 297 298
} SSuperTable;

typedef struct {
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
    char     name[TSDB_DB_NAME_LEN];
    char     create_time[32];
    int64_t  ntables;
    int32_t  vgroups;
    int16_t  replica;
    int16_t  quorum;
    int16_t  days;
    char     keeplist[32];
    int32_t  cache; //MB
    int32_t  blocks;
    int32_t  minrows;
    int32_t  maxrows;
    int8_t   wallevel;
    int32_t  fsync;
    int8_t   comp;
    int8_t   cachelast;
    char     precision[8];   // time resolution
    int8_t   update;
    char     status[16];
318 319
} SDbInfo;

320
typedef struct SDbCfg_S {
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
    //  int       maxtablesPerVnode;
    uint32_t  minRows;        // 0 means default
    uint32_t  maxRows;        // 0 means default
    int       comp;
    int       walLevel;
    int       cacheLast;
    int       fsync;
    int       replica;
    int       update;
    int       keep;
    int       days;
    int       cache;
    int       blocks;
    int       quorum;
    char      precision[8];
336 337 338
} SDbCfg;

typedef struct SDataBase_S {
339 340 341 342 343
    char         dbName[TSDB_DB_NAME_LEN];
    bool         drop;  // 0: use exists, 1: if exists, drop then new create
    SDbCfg       dbCfg;
    uint64_t     superTblCount;
    SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
344 345 346
} SDataBase;

typedef struct SDbs_S {
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
    char        cfgDir[MAX_FILE_NAME_LEN];
    char        host[MAX_HOSTNAME_SIZE];
    struct      sockaddr_in serv_addr;

    uint16_t    port;
    char        user[MAX_USERNAME_SIZE];
    char        password[MAX_PASSWORD_SIZE];
    char        resultFile[MAX_FILE_NAME_LEN];
    bool        use_metric;
    bool        insert_only;
    bool        do_aggreFunc;
    bool        asyncMode;

    uint32_t    threadCount;
    uint32_t    threadCountByCreateTbl;
    uint32_t    dbCount;
    SDataBase   db[MAX_DB_COUNT];

    // statistics
    uint64_t    totalInsertRows;
    uint64_t    totalAffectedRows;
368

369 370
} SDbs;

371
typedef struct SpecifiedQueryInfo_S {
372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
    uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
    uint32_t     concurrent;
    int          sqlCount;
    uint32_t     asyncMode; // 0: sync, 1: async
    uint64_t     subscribeInterval; // ms
    uint64_t     queryTimes;
    bool         subscribeRestart;
    int          subscribeKeepProgress;
    char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
    char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN];
    int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
    int          endAfterConsume[MAX_QUERY_SQL_COUNT];
    TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
    char         topic[MAX_QUERY_SQL_COUNT][32];
    int          consumed[MAX_QUERY_SQL_COUNT];
    TAOS_RES*    res[MAX_QUERY_SQL_COUNT];
    uint64_t     totalQueried;
389
} SpecifiedQueryInfo;
390

391
typedef struct SuperQueryInfo_S {
392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410
    char         sTblName[TSDB_TABLE_NAME_LEN];
    uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
    uint32_t     threadCnt;
    uint32_t     asyncMode; // 0: sync, 1: async
    uint64_t     subscribeInterval; // ms
    bool         subscribeRestart;
    int          subscribeKeepProgress;
    uint64_t     queryTimes;
    int64_t      childTblCount;
    char         childTblPrefix[TSDB_TABLE_NAME_LEN - 20];    // 20 characters reserved for seq
    int          sqlCount;
    char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
    char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN];
    int          resubAfterConsume;
    int          endAfterConsume;
    TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];

    char*        childTblName;
    uint64_t     totalQueried;
411
} SuperQueryInfo;
412 413

typedef struct SQueryMetaInfo_S {
414 415 416 417 418 419 420 421 422 423 424 425
    char         cfgDir[MAX_FILE_NAME_LEN];
    char         host[MAX_HOSTNAME_SIZE];
    uint16_t     port;
    struct       sockaddr_in serv_addr;
    char         user[MAX_USERNAME_SIZE];
    char         password[MAX_PASSWORD_SIZE];
    char         dbName[TSDB_DB_NAME_LEN];
    char         queryMode[MAX_TB_NAME_SIZE];  // taosc, rest

    SpecifiedQueryInfo  specifiedQueryInfo;
    SuperQueryInfo      superQueryInfo;
    uint64_t     totalQueried;
426 427 428
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
    TAOS *    taos;
    TAOS_STMT *stmt;
    int       threadID;
    char      db_name[TSDB_DB_NAME_LEN];
    uint32_t  time_precision;
    char      filePath[4096];
    FILE      *fp;
    char      tb_prefix[TSDB_TABLE_NAME_LEN];
    uint64_t  start_table_from;
    uint64_t  end_table_to;
    int64_t   ntables;
    uint64_t  data_of_rate;
    int64_t   start_time;
    char*     cols;
    bool      use_metric;
    SSuperTable* superTblInfo;
    char      *buffer;    // sql cmd buffer

    // for async insert
    tsem_t    lock_sem;
    int64_t   counter;
    uint64_t  st;
    uint64_t  et;
    uint64_t  lastTs;

    // sample data
    int64_t   samplePos;
    // statistics
    uint64_t  totalInsertRows;
    uint64_t  totalAffectedRows;

    // insert delay statistics
    uint64_t  cntDelay;
    uint64_t  totalDelay;
    uint64_t  avgDelay;
    uint64_t  maxDelay;
    uint64_t  minDelay;

    // seq of query or subscribe
    uint64_t  querySeq;   // sequence number of sql command
    TAOS_SUB*  tsub;
470

471 472
} threadInfo;

473
#ifdef WINDOWS
474 475
#define _CRT_RAND_S

476
#include <windows.h>
477 478 479 480 481
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
482 483
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
484 485
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
486 487 488 489

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

490
static void setupForAnsiEscape(void) {
491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
509
  }
510 511
}

512
static void resetAfterAnsiEscape(void) {
513
  // Reset colors
514
  printf("\x1b[0m");
515 516 517 518 519 520

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
521

522
static int taosRandom()
523 524 525 526 527 528
{
    int number;
    rand_s(&number);

    return number;
}
529
#else   // Not windows
530
static void setupForAnsiEscape(void) {}
531

532
static void resetAfterAnsiEscape(void) {
533 534 535
  // Reset colors
  printf("\x1b[0m");
}
536

537 538
#include <time.h>

539
static int taosRandom()
540
{
541
  return rand();
542 543
}

544
#endif // ifdef Windows
545

546
static void prompt();
547
static int createDatabasesAndStables();
548
static void createChildTables();
549
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet);
550
static int postProceSql(char *host, struct sockaddr_in *pServAddr,
551
        uint16_t port, char* sqlstr, threadInfo *pThreadInfo);
552 553
static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq,
        int disorderRatio, int disorderRange);
554 555
static bool getInfoFromJsonFile(char* file);
static void init_rand_data();
556 557 558 559 560 561 562

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
563
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)",
564 565
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

566 567
#define DEFAULT_DATATYPE_NUM    3

568
SArguments g_args = {
569 570 571 572
    NULL,            // metaFile
    0,               // test_mode
    "127.0.0.1",     // host
    6030,            // port
573
    INTERFACE_BUT,   // iface
574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597
    "root",          // user
#ifdef _TD_POWER_
    "powerdb",      // password
#elif (_TD_TQ_ == true)
    "tqueue",      // password
#else
    "taosdata",      // password
#endif
    "test",          // database
    1,               // replica
    "d",             // tb_prefix
    NULL,            // sqlFile
    true,            // use_metric
    true,            // drop_database
    true,            // insert_only
    false,           // debug_print
    false,           // verbose_print
    false,           // performance statistic print
    false,           // answer_yes;
    "./output.txt",  // output_file
    0,               // mode : sync or async
    {
        "FLOAT",         // datatype
        "INT",           // datatype
598
        "FLOAT",         // datatype. DEFAULT_DATATYPE_NUM is 3
599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
    },
    16,              // len_of_binary
    4,               // num_of_CPR
    10,              // num_of_connections/thread
    0,               // insert_interval
    1,               // query_times
    0,               // interlace_rows;
    30000,           // num_of_RPR
    (1024*1024),     // max_sql_len
    10000,           // num_of_tables
    10000,           // num_of_DPT
    0,               // abort
    0,               // disorderRatio
    1000,            // disorderRange
    1,               // method_of_delete
    NULL,            // arg_list
    0,               // totalInsertRows;
    0,               // totalAffectedRows;
    true,            // demo_mode;
618 619 620 621 622
};



static SDbs            g_Dbs;
623
static int64_t         g_totalChildTables = 0;
624 625 626
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

627 628 629 630
#if _MSC_VER <= 1900
#define __func__ __FUNCTION__
#endif

631
#define debugPrint(fmt, ...) \
632
    do { if (g_args.debug_print || g_args.verbose_print) \
633
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
634

635
#define verbosePrint(fmt, ...) \
636 637
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
638

639 640
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
641
        fprintf(stderr, "PERF: "fmt, __VA_ARGS__); } while(0)
642

643 644 645
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)

646 647 648
// for strncpy buffer overflow
#define min(a, b) (((a) < (b)) ? (a) : (b))

649

650
///////////////////////////////////////////////////
H
hzcheng 已提交
651

652 653
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

654 655 656 657 658 659 660 661
#ifndef TAOSDEMO_COMMIT_SHA1
#define TAOSDEMO_COMMIT_SHA1 "unknown"
#endif

#ifndef TD_VERNUMBER
#define TD_VERNUMBER    "unknown"
#endif

662
#ifndef TAOSDEMO_STATUS
663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679
#define TAOSDEMO_STATUS "unknown"
#endif

static void printVersion() {
    char tdengine_ver[] = TD_VERNUMBER;
    char taosdemo_ver[] = TAOSDEMO_COMMIT_SHA1;
    char taosdemo_status[] = TAOSDEMO_STATUS;

    if (strlen(taosdemo_status) == 0) {
        printf("taosdemo verison %s-%s\n",
                tdengine_ver, taosdemo_ver);
    } else {
        printf("taosdemo verison %s-%s, status:%s\n",
                tdengine_ver, taosdemo_ver, taosdemo_status);
    }
}

680
static void printHelp() {
681 682 683 684 685
    char indent[10] = "        ";
    printf("%s%s%s%s\n", indent, "-f", indent,
            "The meta file to the execution procedure. Default is './meta.json'.");
    printf("%s%s%s%s\n", indent, "-u", indent,
            "The TDengine user name to use when connecting to the server. Default is 'root'.");
686
#ifdef _TD_POWER_
687 688 689 690
    printf("%s%s%s%s\n", indent, "-P", indent,
            "The password to use when connecting to the server. Default is 'powerdb'.");
    printf("%s%s%s%s\n", indent, "-c", indent,
            "Configuration directory. Default is '/etc/power/'.");
691
#elif (_TD_TQ_ == true)
692 693 694 695
    printf("%s%s%s%s\n", indent, "-P", indent,
            "The password to use when connecting to the server. Default is 'tqueue'.");
    printf("%s%s%s%s\n", indent, "-c", indent,
            "Configuration directory. Default is '/etc/tq/'.");
696
#else
697 698 699 700
    printf("%s%s%s%s\n", indent, "-P", indent,
            "The password to use when connecting to the server. Default is 'taosdata'.");
    printf("%s%s%s%s\n", indent, "-c", indent,
            "Configuration directory. Default is '/etc/taos/'.");
701
#endif
702 703 704 705 706
    printf("%s%s%s%s\n", indent, "-h", indent,
            "The host to connect to TDengine. Default is localhost.");
    printf("%s%s%s%s\n", indent, "-p", indent,
            "The TCP/IP port number to use for the connection. Default is 0.");
    printf("%s%s%s%s\n", indent, "-I", indent,
707
#if STMT_IFACE_ENABLED == 1
708
            "The interface (taosc, rest, and stmt) taosdemo uses. Default is 'taosc'.");
709
#else
710
    "The interface (taosc, rest) taosdemo uses. Default is 'taosc'.");
711
#endif
712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727
    printf("%s%s%s%s\n", indent, "-d", indent,
            "Destination database. Default is 'test'.");
    printf("%s%s%s%s\n", indent, "-a", indent,
            "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
    printf("%s%s%s%s\n", indent, "-m", indent,
            "Table prefix name. Default is 'd'.");
    printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
    printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
    printf("%s%s%s%s\n", indent, "-o", indent,
            "Direct output to the named file. Default is './output.txt'.");
    printf("%s%s%s%s\n", indent, "-q", indent,
            "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC.");
    printf("%s%s%s%s\n", indent, "-b", indent,
            "The data_type of columns, default: FLOAT, INT, FLOAT.");
    printf("%s%s%s%s\n", indent, "-w", indent,
            "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
728 729 730 731 732 733 734
    printf("%s%s%s%s%d%s%d\n", indent, "-l", indent,
            "The number of columns per record. Default is ",
            DEFAULT_DATATYPE_NUM,
            ". Max values is ",
            MAX_NUM_COLUMNS);
    printf("%s%s%s%s\n", indent, indent, indent,
            "All of the new column(s) type is INT. If use -b to specify column type, -l will be ignored.");
735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762
    printf("%s%s%s%s\n", indent, "-T", indent,
            "The number of threads. Default is 10.");
    printf("%s%s%s%s\n", indent, "-i", indent,
            "The sleep time (ms) between insertion. Default is 0.");
    printf("%s%s%s%s\n", indent, "-r", indent,
            "The number of records per request. Default is 30000.");
    printf("%s%s%s%s\n", indent, "-t", indent,
            "The number of tables. Default is 10000.");
    printf("%s%s%s%s\n", indent, "-n", indent,
            "The number of records per table. Default is 10000.");
    printf("%s%s%s%s\n", indent, "-M", indent,
            "The value of records generated are totally random.");
    printf("%s%s%s%s\n", indent, indent, indent,
            " The default is to simulate power equipment senario.");
    printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
    printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
    printf("%s%s%s%s\n", indent, "-O", indent,
            "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order.");
    printf("%s%s%s%s\n", indent, "-R", indent,
            "Out of order data's range, ms, default is 1000.");
    printf("%s%s%s%s\n", indent, "-g", indent,
            "Print debug info.");
    printf("%s%s%s\n", indent, "-V, --version\t",
            "Print version info.");
    printf("%s%s%s%s\n", indent, "--help\t", indent,
            "Print command line arguments list info.");
    /*    printf("%s%s%s%s\n", indent, "-D", indent,
          "Delete database if exists. 0: no, 1: yes, default is 1");
763 764 765
          */
}

766 767
static bool isStringNumber(char *input)
{
768 769 770 771
    int len = strlen(input);
    if (0 == len) {
        return false;
    }
772

773 774 775 776
    for (int i = 0; i < len; i++) {
        if (!isdigit(input[i]))
            return false;
    }
777

778
    return true;
779 780
}

781
static void parse_args(int argc, char *argv[], SArguments *arguments) {
782

783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819
    for (int i = 1; i < argc; i++) {
        if (strcmp(argv[i], "-f") == 0) {
            arguments->demo_mode = false;
            arguments->metaFile = argv[++i];
        } else if (strcmp(argv[i], "-c") == 0) {
            if (argc == i+1) {
                printHelp();
                errorPrint("%s", "\n\t-c need a valid path following!\n");
                exit(EXIT_FAILURE);
            }
            tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN);
        } else if (strcmp(argv[i], "-h") == 0) {
            if (argc == i+1) {
                printHelp();
                errorPrint("%s", "\n\t-h need a valid string following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->host = argv[++i];
        } else if (strcmp(argv[i], "-p") == 0) {
            if ((argc == i+1) ||
                    (!isStringNumber(argv[i+1]))) {
                printHelp();
                errorPrint("%s", "\n\t-p need a number following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->port = atoi(argv[++i]);
        } else if (strcmp(argv[i], "-I") == 0) {
            if (argc == i+1) {
                printHelp();
                errorPrint("%s", "\n\t-I need a valid string following!\n");
                exit(EXIT_FAILURE);
            }
            ++i;
            if (0 == strcasecmp(argv[i], "taosc")) {
                arguments->iface = TAOSC_IFACE;
            } else if (0 == strcasecmp(argv[i], "rest")) {
                arguments->iface = REST_IFACE;
820
#if STMT_IFACE_ENABLED == 1
821 822
            } else if (0 == strcasecmp(argv[i], "stmt")) {
                arguments->iface = STMT_IFACE;
823
#endif
824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936
            } else {
                errorPrint("%s", "\n\t-I need a valid string following!\n");
                exit(EXIT_FAILURE);
            }
        } else if (strcmp(argv[i], "-u") == 0) {
            if (argc == i+1) {
                printHelp();
                errorPrint("%s", "\n\t-u need a valid string following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->user = argv[++i];
        } else if (strcmp(argv[i], "-P") == 0) {
            if (argc == i+1) {
                printHelp();
                errorPrint("%s", "\n\t-P need a valid string following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->password = argv[++i];
        } else if (strcmp(argv[i], "-o") == 0) {
            if (argc == i+1) {
                printHelp();
                errorPrint("%s", "\n\t-o need a valid string following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->output_file = argv[++i];
        } else if (strcmp(argv[i], "-s") == 0) {
            if (argc == i+1) {
                printHelp();
                errorPrint("%s", "\n\t-s need a valid string following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->sqlFile = argv[++i];
        } else if (strcmp(argv[i], "-q") == 0) {
            if ((argc == i+1)
                    || (!isStringNumber(argv[i+1]))) {
                printHelp();
                errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, not-0: ASYNC. Default is SYNC.\n");
                exit(EXIT_FAILURE);
            }
            arguments->async_mode = atoi(argv[++i]);
        } else if (strcmp(argv[i], "-T") == 0) {
            if ((argc == i+1)
                    || (!isStringNumber(argv[i+1]))) {
                printHelp();
                errorPrint("%s", "\n\t-T need a number following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->num_of_threads = atoi(argv[++i]);
        } else if (strcmp(argv[i], "-i") == 0) {
            if ((argc == i+1) ||
                    (!isStringNumber(argv[i+1]))) {
                printHelp();
                errorPrint("%s", "\n\t-i need a number following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->insert_interval = atoi(argv[++i]);
        } else if (strcmp(argv[i], "-qt") == 0) {
            if ((argc == i+1)
                    || (!isStringNumber(argv[i+1]))) {
                printHelp();
                errorPrint("%s", "\n\t-qt need a number following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->query_times = atoi(argv[++i]);
        } else if (strcmp(argv[i], "-B") == 0) {
            if ((argc == i+1)
                    || (!isStringNumber(argv[i+1]))) {
                printHelp();
                errorPrint("%s", "\n\t-B need a number following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->interlace_rows = atoi(argv[++i]);
        } else if (strcmp(argv[i], "-r") == 0) {
            if ((argc == i+1)
                    || (!isStringNumber(argv[i+1]))) {
                printHelp();
                errorPrint("%s", "\n\t-r need a number following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->num_of_RPR = atoi(argv[++i]);
        } else if (strcmp(argv[i], "-t") == 0) {
            if ((argc == i+1) ||
                    (!isStringNumber(argv[i+1]))) {
                printHelp();
                errorPrint("%s", "\n\t-t need a number following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->num_of_tables = atoi(argv[++i]);
        } else if (strcmp(argv[i], "-n") == 0) {
            if ((argc == i+1) ||
                    (!isStringNumber(argv[i+1]))) {
                printHelp();
                errorPrint("%s", "\n\t-n need a number following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->num_of_DPT = atoi(argv[++i]);
        } else if (strcmp(argv[i], "-d") == 0) {
            if (argc == i+1) {
                printHelp();
                errorPrint("%s", "\n\t-d need a valid string following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->database = argv[++i];
        } else if (strcmp(argv[i], "-l") == 0) {
            arguments->demo_mode = false;
            if (argc == i+1) {
                if (!isStringNumber(argv[i+1])) {
                    printHelp();
                    errorPrint("%s", "\n\t-l need a number following!\n");
                    exit(EXIT_FAILURE);
                }
            }
            arguments->num_of_CPR = atoi(argv[++i]);
937

938 939
            if (arguments->num_of_CPR > MAX_NUM_COLUMNS) {
                printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_COLUMNS);
940
                prompt();
941
                arguments->num_of_CPR = MAX_NUM_COLUMNS;
942
            }
943

944 945 946 947
            for (int col = DEFAULT_DATATYPE_NUM; col < arguments->num_of_CPR; col ++) {
                arguments->datatype[col] = "INT";
            }
            for (int col = arguments->num_of_CPR; col < MAX_NUM_COLUMNS; col++) {
948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998
                arguments->datatype[col] = NULL;
            }
        } else if (strcmp(argv[i], "-b") == 0) {
            arguments->demo_mode = false;
            if (argc == i+1) {
                printHelp();
                errorPrint("%s", "\n\t-b need valid string following!\n");
                exit(EXIT_FAILURE);
            }
            ++i;
            if (strstr(argv[i], ",") == NULL) {
                // only one col
                if (strcasecmp(argv[i], "INT")
                        && strcasecmp(argv[i], "FLOAT")
                        && strcasecmp(argv[i], "TINYINT")
                        && strcasecmp(argv[i], "BOOL")
                        && strcasecmp(argv[i], "SMALLINT")
                        && strcasecmp(argv[i], "BIGINT")
                        && strcasecmp(argv[i], "DOUBLE")
                        && strcasecmp(argv[i], "BINARY")
                        && strcasecmp(argv[i], "TIMESTAMP")
                        && strcasecmp(argv[i], "NCHAR")) {
                    printHelp();
                    errorPrint("%s", "-b: Invalid data_type!\n");
                    exit(EXIT_FAILURE);
                }
                arguments->datatype[0] = argv[i];
            } else {
                // more than one col
                int index = 0;
                g_dupstr = strdup(argv[i]);
                char *running = g_dupstr;
                char *token = strsep(&running, ",");
                while(token != NULL) {
                    if (strcasecmp(token, "INT")
                            && strcasecmp(token, "FLOAT")
                            && strcasecmp(token, "TINYINT")
                            && strcasecmp(token, "BOOL")
                            && strcasecmp(token, "SMALLINT")
                            && strcasecmp(token, "BIGINT")
                            && strcasecmp(token, "DOUBLE")
                            && strcasecmp(token, "BINARY")
                            && strcasecmp(token, "TIMESTAMP")
                            && strcasecmp(token, "NCHAR")) {
                        printHelp();
                        free(g_dupstr);
                        errorPrint("%s", "-b: Invalid data_type!\n");
                        exit(EXIT_FAILURE);
                    }
                    arguments->datatype[index++] = token;
                    token = strsep(&running, ",");
999
                    if (index >= MAX_NUM_COLUMNS) break;
1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039
                }
                arguments->datatype[index] = NULL;
            }
        } else if (strcmp(argv[i], "-w") == 0) {
            if ((argc == i+1) ||
                    (!isStringNumber(argv[i+1]))) {
                printHelp();
                errorPrint("%s", "\n\t-w need a number following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->len_of_binary = atoi(argv[++i]);
        } else if (strcmp(argv[i], "-m") == 0) {
            if ((argc == i+1) ||
                    (isStringNumber(argv[i+1]))) {
                printHelp();
                errorPrint("%s", "\n\t-m need a letter-initial string following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->tb_prefix = argv[++i];
        } else if (strcmp(argv[i], "-N") == 0) {
            arguments->use_metric = false;
        } else if (strcmp(argv[i], "-M") == 0) {
            arguments->demo_mode = false;
        } else if (strcmp(argv[i], "-x") == 0) {
            arguments->insert_only = false;
        } else if (strcmp(argv[i], "-y") == 0) {
            arguments->answer_yes = true;
        } else if (strcmp(argv[i], "-g") == 0) {
            arguments->debug_print = true;
        } else if (strcmp(argv[i], "-gg") == 0) {
            arguments->verbose_print = true;
        } else if (strcmp(argv[i], "-pp") == 0) {
            arguments->performance_print = true;
        } else if (strcmp(argv[i], "-O") == 0) {
            if ((argc == i+1) ||
                    (!isStringNumber(argv[i+1]))) {
                printHelp();
                errorPrint("%s", "\n\t-O need a number following!\n");
                exit(EXIT_FAILURE);
            }
1040

1041
            arguments->disorderRatio = atoi(argv[++i]);
1042

1043 1044 1045
            if (arguments->disorderRatio > 50) {
                arguments->disorderRatio = 50;
            }
1046

1047 1048 1049
            if (arguments->disorderRatio < 0) {
                arguments->disorderRatio = 0;
            }
1050

1051 1052 1053 1054 1055 1056 1057
        } else if (strcmp(argv[i], "-R") == 0) {
            if ((argc == i+1) ||
                    (!isStringNumber(argv[i+1]))) {
                printHelp();
                errorPrint("%s", "\n\t-R need a number following!\n");
                exit(EXIT_FAILURE);
            }
1058

1059 1060 1061
            arguments->disorderRange = atoi(argv[++i]);
            if (arguments->disorderRange < 0)
                arguments->disorderRange = 1000;
1062

1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091
        } else if (strcmp(argv[i], "-a") == 0) {
            if ((argc == i+1) ||
                    (!isStringNumber(argv[i+1]))) {
                printHelp();
                errorPrint("%s", "\n\t-a need a number following!\n");
                exit(EXIT_FAILURE);
            }
            arguments->replica = atoi(argv[++i]);
            if (arguments->replica > 3 || arguments->replica < 1) {
                arguments->replica = 1;
            }
        } else if (strcmp(argv[i], "-D") == 0) {
            arguments->method_of_delete = atoi(argv[++i]);
            if (arguments->method_of_delete > 3) {
                errorPrint("%s", "\n\t-D need a valud (0~3) number following!\n");
                exit(EXIT_FAILURE);
            }
        } else if ((strcmp(argv[i], "--version") == 0) ||
                (strcmp(argv[i], "-V") == 0)){
            printVersion();
            exit(0);
        } else if (strcmp(argv[i], "--help") == 0) {
            printHelp();
            exit(0);
        } else {
            printHelp();
            errorPrint("%s", "ERROR: wrong options\n");
            exit(EXIT_FAILURE);
        }
S
TD-1057  
Shengliang Guan 已提交
1092
    }
1093

1094
    int columnCount;
1095
    for (columnCount = 0; columnCount < MAX_NUM_COLUMNS; columnCount ++) {
1096 1097 1098
        if (g_args.datatype[columnCount] == NULL) {
            break;
        }
1099 1100
    }

1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119
    if (0 == columnCount) {
        perror("data type error!");
        exit(-1);
    }
    g_args.num_of_CPR = columnCount;

    if (((arguments->debug_print) && (arguments->metaFile == NULL))
            || arguments->verbose_print) {
        printf("###################################################################\n");
        printf("# meta file:                         %s\n", arguments->metaFile);
        printf("# Server IP:                         %s:%hu\n",
                arguments->host == NULL ? "localhost" : arguments->host,
                arguments->port );
        printf("# User:                              %s\n", arguments->user);
        printf("# Password:                          %s\n", arguments->password);
        printf("# Use metric:                        %s\n",
                arguments->use_metric ? "true" : "false");
        if (*(arguments->datatype)) {
            printf("# Specified data type:               ");
1120
            for (int i = 0; i < MAX_NUM_COLUMNS; i++)
1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151
                if (arguments->datatype[i])
                    printf("%s,", arguments->datatype[i]);
                else
                    break;
            printf("\n");
        }
        printf("# Insertion interval:                %"PRIu64"\n",
                arguments->insert_interval);
        printf("# Number of records per req:         %u\n",
                arguments->num_of_RPR);
        printf("# Max SQL length:                    %"PRIu64"\n",
                arguments->max_sql_len);
        printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
        printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
        printf("# Number of Tables:                  %"PRId64"\n",
                arguments->num_of_tables);
        printf("# Number of Data per Table:          %"PRId64"\n",
                arguments->num_of_DPT);
        printf("# Database name:                     %s\n", arguments->database);
        printf("# Table prefix:                      %s\n", arguments->tb_prefix);
        if (arguments->disorderRatio) {
            printf("# Data order:                        %d\n", arguments->disorderRatio);
            printf("# Data out of order rate:            %d\n", arguments->disorderRange);
        }
        printf("# Delete method:                     %d\n", arguments->method_of_delete);
        printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
        printf("# Print debug info:                  %d\n", arguments->debug_print);
        printf("# Print verbose info:                %d\n", arguments->verbose_print);
        printf("###################################################################\n");

        prompt();
1152
    }
1153
}
H
hzcheng 已提交
1154

1155
static void tmfclose(FILE *fp) {
1156 1157 1158
    if (NULL != fp) {
        fclose(fp);
    }
1159
}
1160

1161
static void tmfree(char *buf) {
1162 1163 1164
    if (NULL != buf) {
        free(buf);
    }
1165
}
H
Hui Li 已提交
1166

1167
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
1168 1169 1170 1171 1172 1173 1174 1175 1176
    int i;
    TAOS_RES *res = NULL;
    int32_t   code = -1;

    for (i = 0; i < 5 /* retry */; i++) {
        if (NULL != res) {
            taos_free_result(res);
            res = NULL;
        }
1177

1178 1179 1180 1181 1182
        res = taos_query(taos, command);
        code = taos_errno(res);
        if (0 == code) {
            break;
        }
H
hzcheng 已提交
1183
    }
1184

1185 1186 1187 1188 1189 1190 1191 1192 1193
    verbosePrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
    if (code != 0) {
        if (!quiet) {
            errorPrint("Failed to execute %s, reason: %s\n",
                    command, taos_errstr(res));
        }
        taos_free_result(res);
        //taos_close(taos);
        return -1;
1194
    }
H
hzcheng 已提交
1195

1196 1197 1198 1199
    if (INSERT_TYPE == type) {
        int affectedRows = taos_affected_rows(res);
        taos_free_result(res);
        return affectedRows;
1200
    }
H
hzcheng 已提交
1201

1202
    taos_free_result(res);
1203
    return 0;
1204
}
H
hzcheng 已提交
1205

1206
static void appendResultBufToFile(char *resultBuf, threadInfo *pThreadInfo)
1207
{
1208 1209 1210 1211 1212 1213 1214
    pThreadInfo->fp = fopen(pThreadInfo->filePath, "at");
    if (pThreadInfo->fp == NULL) {
        errorPrint(
                "%s() LN%d, failed to open result file: %s, result will not save to file\n",
                __func__, __LINE__, pThreadInfo->filePath);
        return;
    }
1215

1216 1217 1218
    fprintf(pThreadInfo->fp, "%s", resultBuf);
    tmfclose(pThreadInfo->fp);
    pThreadInfo->fp = NULL;
1219 1220
}

1221
static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) {
1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232
    TAOS_ROW    row = NULL;
    int         num_rows = 0;
    int         num_fields = taos_field_count(res);
    TAOS_FIELD *fields     = taos_fetch_fields(res);

    char* databuf = (char*) calloc(1, 100*1024*1024);
    if (databuf == NULL) {
        errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n",
                __func__, __LINE__);
        return ;
    }
H
hzcheng 已提交
1233

1234
    int64_t   totalLen = 0;
H
hzcheng 已提交
1235

1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260
    // fetch the records row by row
    while((row = taos_fetch_row(res))) {
        if (totalLen >= 100*1024*1024 - 32000) {
            if (strlen(pThreadInfo->filePath) > 0)
                appendResultBufToFile(databuf, pThreadInfo);
            totalLen = 0;
            memset(databuf, 0, 100*1024*1024);
        }
        num_rows++;
        char  temp[16000] = {0};
        int len = taos_print_row(temp, row, fields, num_fields);
        len += sprintf(temp + len, "\n");
        //printf("query result:%s\n", temp);
        memcpy(databuf + totalLen, temp, len);
        totalLen += len;
        verbosePrint("%s() LN%d, totalLen: %"PRId64"\n",
                __func__, __LINE__, totalLen);
    }

    verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n",
            __func__, __LINE__, databuf, pThreadInfo->filePath);
    if (strlen(pThreadInfo->filePath) > 0) {
        appendResultBufToFile(databuf, pThreadInfo);
    }
    free(databuf);
1261
}
H
hzcheng 已提交
1262

1263
static void selectAndGetResult(
1264 1265
        threadInfo *pThreadInfo, char *command)
{
1266 1267 1268 1269 1270 1271 1272 1273
    if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) {
        TAOS_RES *res = taos_query(pThreadInfo->taos, command);
        if (res == NULL || taos_errno(res) != 0) {
            errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n",
                    __func__, __LINE__, command, taos_errstr(res));
            taos_free_result(res);
            return;
        }
1274

1275 1276
        fetchResult(res, pThreadInfo);
        taos_free_result(res);
1277

1278 1279 1280 1281 1282 1283 1284 1285
    } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
        int retCode = postProceSql(
                g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port,
                command,
                pThreadInfo);
        if (0 != retCode) {
            printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID);
        }
1286

1287 1288 1289 1290
    } else {
        errorPrint("%s() LN%d, unknown query mode: %s\n",
                __func__, __LINE__, g_queryInfo.queryMode);
    }
1291
}
H
hzcheng 已提交
1292

1293
static int32_t rand_bool(){
1294 1295 1296 1297
    static int cursor;
    cursor++;
    cursor = cursor % MAX_PREPARED_RAND;
    return randint[cursor] % 2;
1298 1299 1300
}

static int32_t rand_tinyint(){
1301 1302 1303 1304
    static int cursor;
    cursor++;
    cursor = cursor % MAX_PREPARED_RAND;
    return randint[cursor] % 128;
1305 1306 1307
}

static int32_t rand_smallint(){
1308 1309 1310 1311
    static int cursor;
    cursor++;
    cursor = cursor % MAX_PREPARED_RAND;
    return randint[cursor] % 32767;
1312 1313 1314
}

static int32_t rand_int(){
1315 1316 1317 1318
    static int cursor;
    cursor++;
    cursor = cursor % MAX_PREPARED_RAND;
    return randint[cursor];
1319 1320 1321
}

static int64_t rand_bigint(){
1322 1323 1324 1325
    static int cursor;
    cursor++;
    cursor = cursor % MAX_PREPARED_RAND;
    return randbigint[cursor];
1326 1327 1328
}

static float rand_float(){
1329 1330 1331 1332 1333 1334 1335 1336 1337 1338
    static int cursor;
    cursor++;
    cursor = cursor % MAX_PREPARED_RAND;
    return randfloat[cursor];
}

static float demo_current_float(){
    static int cursor;
    cursor++;
    cursor = cursor % MAX_PREPARED_RAND;
1339
    return (float)(9.8 + 0.04 * (randint[cursor] % 10) + randfloat[cursor]/1000000000);
1340 1341 1342 1343 1344 1345
}

static int32_t demo_voltage_int(){
    static int cursor;
    cursor++;
    cursor = cursor % MAX_PREPARED_RAND;
1346
    return 215 + randint[cursor] % 10;
1347 1348 1349 1350 1351 1352
}

static float demo_phase_float(){
    static int cursor;
    cursor++;
    cursor = cursor % MAX_PREPARED_RAND;
1353
    return (float)((115 + randint[cursor] % 10 + randfloat[cursor]/1000000000)/360);
1354 1355
}

1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372
#if 0
static const char charNum[] = "0123456789";

static void nonrand_string(char *, int) __attribute__ ((unused));   // reserve for debugging purpose
static void nonrand_string(char *str, int size)
{
  str[0] = 0;
  if (size > 0) {
    int n;
    for (n = 0; n < size; n++) {
      str[n] = charNum[n % 10];
    }
    str[n] = 0;
  }
}
#endif

1373
static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1374

1375
static void rand_string(char *str, int size) {
1376 1377 1378 1379 1380 1381 1382 1383 1384
    str[0] = 0;
    if (size > 0) {
        //--size;
        int n;
        for (n = 0; n < size; n++) {
            int key = abs(rand_tinyint()) % (int)(sizeof(charset) - 1);
            str[n] = charset[key];
        }
        str[n] = 0;
1385 1386 1387 1388
    }
}

static double rand_double() {
1389 1390 1391 1392
    static int cursor;
    cursor++;
    cursor = cursor % MAX_PREPARED_RAND;
    return randdouble[cursor];
1393 1394 1395
}

static void init_rand_data() {
1396 1397 1398 1399 1400 1401
    for (int i = 0; i < MAX_PREPARED_RAND; i++){
        randint[i] = (int)(taosRandom() % 65535);
        randbigint[i] = (int64_t)(taosRandom() % 2147483648);
        randfloat[i] = (float)(taosRandom() / 1000.0);
        randdouble[i] = (double)(taosRandom() / 1000000.0);
    }
1402 1403
}

1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1424
static int printfInsertMeta() {
1425
    SHOW_PARSE_RESULT_START();
1426

1427 1428 1429 1430 1431
    if (g_args.demo_mode)
        printf("\ntaosdemo is simulating data generated by power equipments monitoring...\n\n");
    else
        printf("\ntaosdemo is simulating random data as you request..\n\n");

1432 1433 1434 1435 1436 1437 1438
    if (g_args.iface != INTERFACE_BUT) {
        // first time if no iface specified
        printf("interface:                  \033[33m%s\033[0m\n",
            (g_args.iface==TAOSC_IFACE)?"taosc":
            (g_args.iface==REST_IFACE)?"rest":"stmt");
    }

1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455
    printf("host:                       \033[33m%s:%u\033[0m\n",
            g_Dbs.host, g_Dbs.port);
    printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
    printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
    printf("configDir:                  \033[33m%s\033[0m\n", configDir);
    printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
    printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
    printf("thread num of create table: \033[33m%d\033[0m\n",
            g_Dbs.threadCountByCreateTbl);
    printf("top insert interval:        \033[33m%"PRIu64"\033[0m\n",
            g_args.insert_interval);
    printf("number of records per req:  \033[33m%u\033[0m\n",
            g_args.num_of_RPR);
    printf("max sql length:             \033[33m%"PRIu64"\033[0m\n",
            g_args.max_sql_len);

    printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
1456

1457 1458 1459 1460 1461 1462 1463 1464 1465
    for (int i = 0; i < g_Dbs.dbCount; i++) {
        printf("database[\033[33m%d\033[0m]:\n", i);
        printf("  database[%d] name:      \033[33m%s\033[0m\n",
                i, g_Dbs.db[i].dbName);
        if (0 == g_Dbs.db[i].drop) {
            printf("  drop:                  \033[33mno\033[0m\n");
        } else {
            printf("  drop:                  \033[33myes\033[0m\n");
        }
1466

1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515
        if (g_Dbs.db[i].dbCfg.blocks > 0) {
            printf("  blocks:                \033[33m%d\033[0m\n",
                    g_Dbs.db[i].dbCfg.blocks);
        }
        if (g_Dbs.db[i].dbCfg.cache > 0) {
            printf("  cache:                 \033[33m%d\033[0m\n",
                    g_Dbs.db[i].dbCfg.cache);
        }
        if (g_Dbs.db[i].dbCfg.days > 0) {
            printf("  days:                  \033[33m%d\033[0m\n",
                    g_Dbs.db[i].dbCfg.days);
        }
        if (g_Dbs.db[i].dbCfg.keep > 0) {
            printf("  keep:                  \033[33m%d\033[0m\n",
                    g_Dbs.db[i].dbCfg.keep);
        }
        if (g_Dbs.db[i].dbCfg.replica > 0) {
            printf("  replica:               \033[33m%d\033[0m\n",
                    g_Dbs.db[i].dbCfg.replica);
        }
        if (g_Dbs.db[i].dbCfg.update > 0) {
            printf("  update:                \033[33m%d\033[0m\n",
                    g_Dbs.db[i].dbCfg.update);
        }
        if (g_Dbs.db[i].dbCfg.minRows > 0) {
            printf("  minRows:               \033[33m%d\033[0m\n",
                    g_Dbs.db[i].dbCfg.minRows);
        }
        if (g_Dbs.db[i].dbCfg.maxRows > 0) {
            printf("  maxRows:               \033[33m%d\033[0m\n",
                    g_Dbs.db[i].dbCfg.maxRows);
        }
        if (g_Dbs.db[i].dbCfg.comp > 0) {
            printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
        }
        if (g_Dbs.db[i].dbCfg.walLevel > 0) {
            printf("  walLevel:              \033[33m%d\033[0m\n",
                    g_Dbs.db[i].dbCfg.walLevel);
        }
        if (g_Dbs.db[i].dbCfg.fsync > 0) {
            printf("  fsync:                 \033[33m%d\033[0m\n",
                    g_Dbs.db[i].dbCfg.fsync);
        }
        if (g_Dbs.db[i].dbCfg.quorum > 0) {
            printf("  quorum:                \033[33m%d\033[0m\n",
                    g_Dbs.db[i].dbCfg.quorum);
        }
        if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
            if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
1516 1517 1518 1519
#if NANO_SECOND_ENABLED == 1
                    || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))
#endif
                    || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ns", 2))) {
1520 1521 1522 1523 1524 1525 1526 1527
                printf("  precision:             \033[33m%s\033[0m\n",
                        g_Dbs.db[i].dbCfg.precision);
            } else {
                printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
                        g_Dbs.db[i].dbCfg.precision);
                return -1;
            }
        }
1528

1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544
        printf("  super table count:     \033[33m%"PRIu64"\033[0m\n",
                g_Dbs.db[i].superTblCount);
        for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
            printf("  super table[\033[33m%"PRIu64"\033[0m]:\n", j);

            printf("      stbName:           \033[33m%s\033[0m\n",
                    g_Dbs.db[i].superTbls[j].sTblName);

            if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
                printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
            } else if (AUTO_CREATE_SUBTBL ==
                    g_Dbs.db[i].superTbls[j].autoCreateTable) {
                printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
            } else {
                printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
            }
1545

1546 1547 1548 1549 1550 1551 1552
            if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
                printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
            } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
                printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
            } else {
                printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
            }
1553

1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
            printf("      childTblCount:     \033[33m%"PRId64"\033[0m\n",
                    g_Dbs.db[i].superTbls[j].childTblCount);
            printf("      childTblPrefix:    \033[33m%s\033[0m\n",
                    g_Dbs.db[i].superTbls[j].childTblPrefix);
            printf("      dataSource:        \033[33m%s\033[0m\n",
                    g_Dbs.db[i].superTbls[j].dataSource);
            printf("      iface:             \033[33m%s\033[0m\n",
                    (g_Dbs.db[i].superTbls[j].iface==TAOSC_IFACE)?"taosc":
                    (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest":"stmt");
            if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
                printf("      childTblLimit:     \033[33m%"PRId64"\033[0m\n",
                        g_Dbs.db[i].superTbls[j].childTblLimit);
            }
            if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) {
                printf("      childTblOffset:    \033[33m%"PRIu64"\033[0m\n",
                        g_Dbs.db[i].superTbls[j].childTblOffset);
            }
            printf("      insertRows:        \033[33m%"PRId64"\033[0m\n",
                    g_Dbs.db[i].superTbls[j].insertRows);
            /*
               if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
               printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
               }else {
               printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
               }
               */
            printf("      interlaceRows:     \033[33m%u\033[0m\n",
                    g_Dbs.db[i].superTbls[j].interlaceRows);

            if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
                printf("      stable insert interval:   \033[33m%"PRIu64"\033[0m\n",
                        g_Dbs.db[i].superTbls[j].insertInterval);
            }
1587

1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638
            printf("      disorderRange:     \033[33m%d\033[0m\n",
                    g_Dbs.db[i].superTbls[j].disorderRange);
            printf("      disorderRatio:     \033[33m%d\033[0m\n",
                    g_Dbs.db[i].superTbls[j].disorderRatio);
            printf("      maxSqlLen:         \033[33m%"PRIu64"\033[0m\n",
                    g_Dbs.db[i].superTbls[j].maxSqlLen);
            printf("      timeStampStep:     \033[33m%"PRId64"\033[0m\n",
                    g_Dbs.db[i].superTbls[j].timeStampStep);
            printf("      startTimestamp:    \033[33m%s\033[0m\n",
                    g_Dbs.db[i].superTbls[j].startTimestamp);
            printf("      sampleFormat:      \033[33m%s\033[0m\n",
                    g_Dbs.db[i].superTbls[j].sampleFormat);
            printf("      sampleFile:        \033[33m%s\033[0m\n",
                    g_Dbs.db[i].superTbls[j].sampleFile);
            printf("      tagsFile:          \033[33m%s\033[0m\n",
                    g_Dbs.db[i].superTbls[j].tagsFile);
            printf("      columnCount:       \033[33m%d\033[0m\n",
                    g_Dbs.db[i].superTbls[j].columnCount);
            for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
                //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
                if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                                "binary", 6))
                        || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                                "nchar", 5))) {
                    printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
                            g_Dbs.db[i].superTbls[j].columns[k].dataType,
                            g_Dbs.db[i].superTbls[j].columns[k].dataLen);
                } else {
                    printf("column[%d]:\033[33m%s\033[0m ", k,
                            g_Dbs.db[i].superTbls[j].columns[k].dataType);
                }
            }
            printf("\n");

            printf("      tagCount:            \033[33m%d\033[0m\n        ",
                    g_Dbs.db[i].superTbls[j].tagCount);
            for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
                //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
                if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                                "binary", strlen("binary")))
                        || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                                "nchar", strlen("nchar")))) {
                    printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
                            g_Dbs.db[i].superTbls[j].tags[k].dataType,
                            g_Dbs.db[i].superTbls[j].tags[k].dataLen);
                } else {
                    printf("tag[%d]:\033[33m%s\033[0m ", k,
                            g_Dbs.db[i].superTbls[j].tags[k].dataType);
                }
            }
            printf("\n");
1639
        }
1640
        printf("\n");
1641
    }
1642

1643
    SHOW_PARSE_RESULT_END();
1644

1645
    return 0;
1646 1647 1648
}

static void printfInsertMetaToFile(FILE* fp) {
1649

1650
    SHOW_PARSE_RESULT_START_TO_FILE(fp);
1651

1652 1653 1654 1655 1656 1657 1658 1659 1660
    fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
    fprintf(fp, "user:                       %s\n", g_Dbs.user);
    fprintf(fp, "configDir:                  %s\n", configDir);
    fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
    fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
    fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
    fprintf(fp, "number of records per req:  %u\n", g_args.num_of_RPR);
    fprintf(fp, "max sql length:             %"PRIu64"\n", g_args.max_sql_len);
    fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
1661

1662 1663 1664 1665 1666 1667 1668 1669
    for (int i = 0; i < g_Dbs.dbCount; i++) {
        fprintf(fp, "database[%d]:\n", i);
        fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
        if (0 == g_Dbs.db[i].drop) {
            fprintf(fp, "  drop:                  no\n");
        }else {
            fprintf(fp, "  drop:                  yes\n");
        }
1670

1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708
        if (g_Dbs.db[i].dbCfg.blocks > 0) {
            fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
        }
        if (g_Dbs.db[i].dbCfg.cache > 0) {
            fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
        }
        if (g_Dbs.db[i].dbCfg.days > 0) {
            fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
        }
        if (g_Dbs.db[i].dbCfg.keep > 0) {
            fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
        }
        if (g_Dbs.db[i].dbCfg.replica > 0) {
            fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
        }
        if (g_Dbs.db[i].dbCfg.update > 0) {
            fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
        }
        if (g_Dbs.db[i].dbCfg.minRows > 0) {
            fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
        }
        if (g_Dbs.db[i].dbCfg.maxRows > 0) {
            fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
        }
        if (g_Dbs.db[i].dbCfg.comp > 0) {
            fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
        }
        if (g_Dbs.db[i].dbCfg.walLevel > 0) {
            fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
        }
        if (g_Dbs.db[i].dbCfg.fsync > 0) {
            fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
        }
        if (g_Dbs.db[i].dbCfg.quorum > 0) {
            fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
        }
        if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
            if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
1709 1710 1711
#if NANO_SECOND_ENABLED == 1
                    || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ns", 2))
#endif
1712 1713 1714 1715 1716 1717 1718 1719
                    || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
                fprintf(fp, "  precision:             %s\n",
                        g_Dbs.db[i].dbCfg.precision);
            } else {
                fprintf(fp, "  precision error:       %s\n",
                        g_Dbs.db[i].dbCfg.precision);
            }
        }
1720

1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736
        fprintf(fp, "  super table count:     %"PRIu64"\n",
                g_Dbs.db[i].superTblCount);
        for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
            fprintf(fp, "  super table[%d]:\n", j);

            fprintf(fp, "      stbName:           %s\n",
                    g_Dbs.db[i].superTbls[j].sTblName);

            if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
                fprintf(fp, "      autoCreateTable:   %s\n",  "no");
            } else if (AUTO_CREATE_SUBTBL
                    == g_Dbs.db[i].superTbls[j].autoCreateTable) {
                fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
            } else {
                fprintf(fp, "      autoCreateTable:   %s\n",  "error");
            }
1737

1738 1739 1740 1741 1742 1743 1744 1745
            if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
                fprintf(fp, "      childTblExists:    %s\n",  "no");
            } else if (TBL_ALREADY_EXISTS
                    == g_Dbs.db[i].superTbls[j].childTblExists) {
                fprintf(fp, "      childTblExists:    %s\n",  "yes");
            } else {
                fprintf(fp, "      childTblExists:    %s\n",  "error");
            }
1746

1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826
            fprintf(fp, "      childTblCount:     %"PRId64"\n",
                    g_Dbs.db[i].superTbls[j].childTblCount);
            fprintf(fp, "      childTblPrefix:    %s\n",
                    g_Dbs.db[i].superTbls[j].childTblPrefix);
            fprintf(fp, "      dataSource:        %s\n",
                    g_Dbs.db[i].superTbls[j].dataSource);
            fprintf(fp, "      iface:             %s\n",
                    (g_Dbs.db[i].superTbls[j].iface==TAOSC_IFACE)?"taosc":
                    (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest":"stmt");
            fprintf(fp, "      insertRows:        %"PRId64"\n",
                    g_Dbs.db[i].superTbls[j].insertRows);
            fprintf(fp, "      interlace rows:    %u\n",
                    g_Dbs.db[i].superTbls[j].interlaceRows);
            if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
                fprintf(fp, "      stable insert interval:   %"PRIu64"\n",
                        g_Dbs.db[i].superTbls[j].insertInterval);
            }
            /*
               if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
               fprintf(fp, "      multiThreadWriteOneTbl:  no\n");
               }else {
               fprintf(fp, "      multiThreadWriteOneTbl:  yes\n");
               }
               */
            fprintf(fp, "      interlaceRows:     %u\n",
                    g_Dbs.db[i].superTbls[j].interlaceRows);
            fprintf(fp, "      disorderRange:     %d\n",
                    g_Dbs.db[i].superTbls[j].disorderRange);
            fprintf(fp, "      disorderRatio:     %d\n",
                    g_Dbs.db[i].superTbls[j].disorderRatio);
            fprintf(fp, "      maxSqlLen:         %"PRIu64"\n",
                    g_Dbs.db[i].superTbls[j].maxSqlLen);

            fprintf(fp, "      timeStampStep:     %"PRId64"\n",
                    g_Dbs.db[i].superTbls[j].timeStampStep);
            fprintf(fp, "      startTimestamp:    %s\n",
                    g_Dbs.db[i].superTbls[j].startTimestamp);
            fprintf(fp, "      sampleFormat:      %s\n",
                    g_Dbs.db[i].superTbls[j].sampleFormat);
            fprintf(fp, "      sampleFile:        %s\n",
                    g_Dbs.db[i].superTbls[j].sampleFile);
            fprintf(fp, "      tagsFile:          %s\n",
                    g_Dbs.db[i].superTbls[j].tagsFile);

            fprintf(fp, "      columnCount:       %d\n        ",
                    g_Dbs.db[i].superTbls[j].columnCount);
            for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
                //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
                if ((0 == strncasecmp(
                                g_Dbs.db[i].superTbls[j].columns[k].dataType,
                                "binary", strlen("binary")))
                        || (0 == strncasecmp(
                                g_Dbs.db[i].superTbls[j].columns[k].dataType,
                                "nchar", strlen("nchar")))) {
                    fprintf(fp, "column[%d]:%s(%d) ", k,
                            g_Dbs.db[i].superTbls[j].columns[k].dataType,
                            g_Dbs.db[i].superTbls[j].columns[k].dataLen);
                } else {
                    fprintf(fp, "column[%d]:%s ",
                            k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
                }
            }
            fprintf(fp, "\n");

            fprintf(fp, "      tagCount:            %d\n        ",
                    g_Dbs.db[i].superTbls[j].tagCount);
            for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
                //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
                if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                                "binary", strlen("binary")))
                        || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                                "nchar", strlen("nchar")))) {
                    fprintf(fp, "tag[%d]:%s(%d) ",
                            k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
                            g_Dbs.db[i].superTbls[j].tags[k].dataLen);
                } else {
                    fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
                }
            }
            fprintf(fp, "\n");
1827
        }
1828
        fprintf(fp, "\n");
1829
    }
1830

1831
    SHOW_PARSE_RESULT_END_TO_FILE(fp);
1832 1833 1834
}

static void printfQueryMeta() {
1835

1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904
    SHOW_PARSE_RESULT_START();

    printf("host:                    \033[33m%s:%u\033[0m\n",
            g_queryInfo.host, g_queryInfo.port);
    printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
    printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

    printf("\n");

    if ((SUBSCRIBE_TEST == g_args.test_mode) || (QUERY_TEST == g_args.test_mode)) {
        printf("specified table query info:                   \n");
        printf("sqlCount:       \033[33m%d\033[0m\n",
                g_queryInfo.specifiedQueryInfo.sqlCount);
        if (g_queryInfo.specifiedQueryInfo.sqlCount > 0) {
            printf("specified tbl query times:\n");
            printf("                \033[33m%"PRIu64"\033[0m\n",
                    g_queryInfo.specifiedQueryInfo.queryTimes);
            printf("query interval: \033[33m%"PRIu64" ms\033[0m\n",
                    g_queryInfo.specifiedQueryInfo.queryInterval);
            printf("top query times:\033[33m%"PRIu64"\033[0m\n", g_args.query_times);
            printf("concurrent:     \033[33m%d\033[0m\n",
                    g_queryInfo.specifiedQueryInfo.concurrent);
            printf("mod:            \033[33m%s\033[0m\n",
                    (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync");
            printf("interval:       \033[33m%"PRIu64"\033[0m\n",
                    g_queryInfo.specifiedQueryInfo.subscribeInterval);
            printf("restart:        \033[33m%d\033[0m\n",
                    g_queryInfo.specifiedQueryInfo.subscribeRestart);
            printf("keepProgress:   \033[33m%d\033[0m\n",
                    g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);

            for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
                printf("  sql[%d]: \033[33m%s\033[0m\n",
                        i, g_queryInfo.specifiedQueryInfo.sql[i]);
            }
            printf("\n");
        }

        printf("super table query info:\n");
        printf("sqlCount:       \033[33m%d\033[0m\n",
                g_queryInfo.superQueryInfo.sqlCount);

        if (g_queryInfo.superQueryInfo.sqlCount > 0) {
            printf("query interval: \033[33m%"PRIu64"\033[0m\n",
                    g_queryInfo.superQueryInfo.queryInterval);
            printf("threadCnt:      \033[33m%d\033[0m\n",
                    g_queryInfo.superQueryInfo.threadCnt);
            printf("childTblCount:  \033[33m%"PRId64"\033[0m\n",
                    g_queryInfo.superQueryInfo.childTblCount);
            printf("stable name:    \033[33m%s\033[0m\n",
                    g_queryInfo.superQueryInfo.sTblName);
            printf("stb query times:\033[33m%"PRIu64"\033[0m\n",
                    g_queryInfo.superQueryInfo.queryTimes);

            printf("mod:            \033[33m%s\033[0m\n",
                    (g_queryInfo.superQueryInfo.asyncMode)?"async":"sync");
            printf("interval:       \033[33m%"PRIu64"\033[0m\n",
                    g_queryInfo.superQueryInfo.subscribeInterval);
            printf("restart:        \033[33m%d\033[0m\n",
                    g_queryInfo.superQueryInfo.subscribeRestart);
            printf("keepProgress:   \033[33m%d\033[0m\n",
                    g_queryInfo.superQueryInfo.subscribeKeepProgress);

            for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
                printf("  sql[%d]: \033[33m%s\033[0m\n",
                        i, g_queryInfo.superQueryInfo.sql[i]);
            }
            printf("\n");
        }
1905
    }
1906

1907
    SHOW_PARSE_RESULT_END();
1908 1909
}

1910
static char* formatTimestamp(char* buf, int64_t val, int precision) {
1911
    time_t tt;
1912
    if (precision == TSDB_TIME_PRECISION_MICRO) {
1913
        tt = (time_t)(val / 1000000);
1914 1915 1916 1917
#if NANO_SECOND_ENABLED == 1
    } if (precision == TSDB_TIME_PRECISION_NANO) {
        tt = (time_t)(val / 1000000000);
#endif
1918 1919 1920
    } else {
        tt = (time_t)(val / 1000);
    }
1921

1922 1923 1924 1925 1926 1927 1928
    /* comment out as it make testcases like select_with_tags.sim fail.
       but in windows, this may cause the call to localtime crash if tt < 0,
       need to find a better solution.
       if (tt < 0) {
       tt = 0;
       }
       */
1929 1930

#ifdef WINDOWS
1931
    if (tt < 0) tt = 0;
1932 1933
#endif

1934 1935
    struct tm* ptm = localtime(&tt);
    size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);
1936

1937
    if (precision == TSDB_TIME_PRECISION_MICRO) {
1938
        sprintf(buf + pos, ".%06d", (int)(val % 1000000));
1939 1940 1941 1942
#if NANO_SECOND_ENABLED == 1
    } else if (precision == TSDB_TIME_PRECISION_NANO) {
        sprintf(buf + pos, ".%09d", (int)(val % 1000000000));
#endif
1943 1944 1945
    } else {
        sprintf(buf + pos, ".%03d", (int)(val % 1000));
    }
1946

1947
    return buf;
1948 1949
}

1950
static void xDumpFieldToFile(FILE* fp, const char* val,
1951
        TAOS_FIELD* field, int32_t length, int precision) {
1952

1953 1954 1955 1956
    if (val == NULL) {
        fprintf(fp, "%s", TSDB_DATA_NULL_STR);
        return;
    }
1957

1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993
    char buf[TSDB_MAX_BYTES_PER_ROW];
    switch (field->type) {
        case TSDB_DATA_TYPE_BOOL:
            fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
            break;
        case TSDB_DATA_TYPE_TINYINT:
            fprintf(fp, "%d", *((int8_t *)val));
            break;
        case TSDB_DATA_TYPE_SMALLINT:
            fprintf(fp, "%d", *((int16_t *)val));
            break;
        case TSDB_DATA_TYPE_INT:
            fprintf(fp, "%d", *((int32_t *)val));
            break;
        case TSDB_DATA_TYPE_BIGINT:
            fprintf(fp, "%" PRId64, *((int64_t *)val));
            break;
        case TSDB_DATA_TYPE_FLOAT:
            fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
            break;
        case TSDB_DATA_TYPE_DOUBLE:
            fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
            break;
        case TSDB_DATA_TYPE_BINARY:
        case TSDB_DATA_TYPE_NCHAR:
            memcpy(buf, val, length);
            buf[length] = 0;
            fprintf(fp, "\'%s\'", buf);
            break;
        case TSDB_DATA_TYPE_TIMESTAMP:
            formatTimestamp(buf, *(int64_t*)val, precision);
            fprintf(fp, "'%s'", buf);
            break;
        default:
            break;
    }
1994 1995 1996
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
1997 1998 1999 2000
    TAOS_ROW row = taos_fetch_row(tres);
    if (row == NULL) {
        return 0;
    }
2001

2002 2003 2004 2005 2006 2007
    FILE* fp = fopen(fname, "at");
    if (fp == NULL) {
        errorPrint("%s() LN%d, failed to open file: %s\n",
                __func__, __LINE__, fname);
        return -1;
    }
2008

2009 2010 2011
    int num_fields = taos_num_fields(tres);
    TAOS_FIELD *fields = taos_fetch_fields(tres);
    int precision = taos_result_precision(tres);
2012

2013 2014 2015 2016 2017
    for (int col = 0; col < num_fields; col++) {
        if (col > 0) {
            fprintf(fp, ",");
        }
        fprintf(fp, "%s", fields[col].name);
2018 2019 2020
    }
    fputc('\n', fp);

2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035
    int numOfRows = 0;
    do {
        int32_t* length = taos_fetch_lengths(tres);
        for (int i = 0; i < num_fields; i++) {
            if (i > 0) {
                fputc(',', fp);
            }
            xDumpFieldToFile(fp,
                    (const char*)row[i], fields +i, length[i], precision);
        }
        fputc('\n', fp);

        numOfRows++;
        row = taos_fetch_row(tres);
    } while( row != NULL);
2036

2037
    fclose(fp);
2038

2039
    return numOfRows;
2040 2041
}

2042
static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
2043 2044 2045
    TAOS_RES * res;
    TAOS_ROW row = NULL;
    int count = 0;
2046

2047 2048
    res = taos_query(taos, "show databases;");
    int32_t code = taos_errno(res);
2049

2050 2051 2052 2053
    if (code != 0) {
        errorPrint( "failed to run <show databases>, reason: %s\n",
                taos_errstr(res));
        return -1;
2054
    }
2055

2056
    TAOS_FIELD *fields = taos_fetch_fields(res);
2057

2058 2059 2060 2061 2062 2063
    while((row = taos_fetch_row(res)) != NULL) {
        // sys database name : 'log'
        if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log",
                    fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) {
            continue;
        }
2064

2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106
        dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
        if (dbInfos[count] == NULL) {
            errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
            return -1;
        }

        tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
                fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
        formatTimestamp(dbInfos[count]->create_time,
                *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
                TSDB_TIME_PRECISION_MILLI);
        dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
        dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
        dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
        dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
        dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);

        tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
                fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
        dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
        dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
        dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
        dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
        dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
        dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
        dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
        dbInfos[count]->cachelast =
            (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));

        tstrncpy(dbInfos[count]->precision,
                (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
                fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
        dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
        tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
                fields[TSDB_SHOW_DB_STATUS_INDEX].bytes);

        count++;
        if (count > MAX_DATABASE_COUNT) {
            errorPrint("%s() LN%d, The database count overflow than %d\n",
                    __func__, __LINE__, MAX_DATABASE_COUNT);
            break;
        }
2107 2108
    }

2109
    return count;
2110 2111
}

2112
static void printfDbInfoForQueryToFile(
2113
        char* filename, SDbInfo* dbInfos, int index) {
2114

2115 2116
    if (filename[0] == 0)
        return;
2117

2118 2119 2120 2121 2122
    FILE *fp = fopen(filename, "at");
    if (fp == NULL) {
        errorPrint( "failed to open file: %s\n", filename);
        return;
    }
2123

2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144
    fprintf(fp, "================ database[%d] ================\n", index);
    fprintf(fp, "name: %s\n", dbInfos->name);
    fprintf(fp, "created_time: %s\n", dbInfos->create_time);
    fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables);
    fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);
    fprintf(fp, "replica: %d\n", dbInfos->replica);
    fprintf(fp, "quorum: %d\n", dbInfos->quorum);
    fprintf(fp, "days: %d\n", dbInfos->days);
    fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);
    fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
    fprintf(fp, "blocks: %d\n", dbInfos->blocks);
    fprintf(fp, "minrows: %d\n", dbInfos->minrows);
    fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
    fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
    fprintf(fp, "fsync: %d\n", dbInfos->fsync);
    fprintf(fp, "comp: %d\n", dbInfos->comp);
    fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);
    fprintf(fp, "precision: %s\n", dbInfos->precision);
    fprintf(fp, "update: %d\n", dbInfos->update);
    fprintf(fp, "status: %s\n", dbInfos->status);
    fprintf(fp, "\n");
2145

2146
    fclose(fp);
2147 2148 2149
}

static void printfQuerySystemInfo(TAOS * taos) {
2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164
    char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
    char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
    TAOS_RES* res;

    time_t t;
    struct tm* lt;
    time(&t);
    lt = localtime(&t);
    snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
            lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
            lt->tm_sec);

    // show variables
    res = taos_query(taos, "show variables;");
    //fetchResult(res, filename);
2165
    xDumpResultToFile(filename, res);
2166

2167 2168
    // show dnodes
    res = taos_query(taos, "show dnodes;");
2169
    xDumpResultToFile(filename, res);
2170
    //fetchResult(res, filename);
2171

2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199
    // show databases
    res = taos_query(taos, "show databases;");
    SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
    if (dbInfos == NULL) {
        errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
        return;
    }
    int dbCount = getDbFromServer(taos, dbInfos);
    if (dbCount <= 0) {
        free(dbInfos);
        return;
    }

    for (int i = 0; i < dbCount; i++) {
        // printf database info
        printfDbInfoForQueryToFile(filename, dbInfos[i], i);

        // show db.vgroups
        snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
        res = taos_query(taos, buffer);
        xDumpResultToFile(filename, res);

        // show db.stables
        snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
        res = taos_query(taos, buffer);
        xDumpResultToFile(filename, res);
        free(dbInfos[i]);
    }
2200

2201
    free(dbInfos);
2202 2203
}

2204
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
2205
        char* sqlstr, threadInfo *pThreadInfo)
2206
{
2207
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
2208

2209
    char *url = "/rest/sql";
2210

2211 2212 2213 2214
    int bytes, sent, received, req_str_len, resp_len;
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
2215

2216
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;
2217

2218 2219 2220 2221 2222
    request_buf = malloc(req_buf_len);
    if (NULL == request_buf) {
      errorPrint("%s", "ERROR, cannot allocate memory.\n");
      exit(EXIT_FAILURE);
    }
2223

2224 2225
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};
2226

2227 2228 2229 2230 2231 2232 2233 2234
    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
2235

2236 2237 2238 2239
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);
2240

2241
    char base64_buf[INPUT_BUF_LEN];
2242
#ifdef WINDOWS
2243
    WSADATA wsaData;
2244
    WSAStartup(MAKEWORD(2, 2), &wsaData);
2245 2246
    SOCKET sockfd;
#else
2247
    int sockfd;
2248
#endif
2249 2250
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (sockfd < 0) {
2251
#ifdef WINDOWS
2252
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
2253
#endif
2254 2255 2256 2257
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
        free(request_buf);
        ERROR_EXIT("ERROR opening socket");
    }
2258

2259
    int retConn = connect(sockfd, (struct sockaddr *)pServAddr, sizeof(struct sockaddr));
2260 2261 2262 2263 2264
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
        free(request_buf);
        ERROR_EXIT("ERROR connecting");
    }
2265

2266
    memset(base64_buf, 0, INPUT_BUF_LEN);
2267

2268 2269 2270 2271 2272 2273 2274 2275
    for (int n = 0, m = 0; n < userpass_buf_len;) {
      uint32_t oct_a = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t oct_b = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t oct_c = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;
2276

2277 2278 2279 2280 2281
      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }
2282

2283 2284
    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';
2285

2286 2287 2288
    debugPrint("%s() LN%d: auth string base64 encoded: %s\n",
            __func__, __LINE__, base64_buf);
    char *auth = base64_buf;
2289

2290 2291 2292 2293 2294 2295 2296 2297 2298
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
2299

2300 2301 2302
    req_str_len = strlen(request_buf);
    sent = 0;
    do {
2303
#ifdef WINDOWS
2304
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
2305
#else
2306
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
2307
#endif
2308 2309 2310 2311 2312 2313
        if (bytes < 0)
            ERROR_EXIT("ERROR writing message to socket");
        if (bytes == 0)
            break;
        sent+=bytes;
    } while(sent < req_str_len);
2314

2315 2316 2317 2318
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
    received = 0;
    do {
2319
#ifdef WINDOWS
2320
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
2321
#else
2322
        bytes = read(sockfd, response_buf + received, resp_len - received);
2323
#endif
2324 2325 2326 2327 2328 2329 2330 2331
        if (bytes < 0) {
            free(request_buf);
            ERROR_EXIT("ERROR reading response from socket");
        }
        if (bytes == 0)
            break;
        received += bytes;
    } while(received < resp_len);
2332

2333 2334 2335 2336
    if (received == resp_len) {
        free(request_buf);
        ERROR_EXIT("ERROR storing complete response from socket");
    }
H
Haojun Liao 已提交
2337

2338 2339
    response_buf[RESP_BUF_LEN - 1] = '\0';
    printf("Response:\n%s\n", response_buf);
H
Haojun Liao 已提交
2340

2341 2342
    if (strlen(pThreadInfo->filePath) > 0) {
       appendResultBufToFile(response_buf, pThreadInfo);
2343 2344
    }

2345
    free(request_buf);
2346
#ifdef WINDOWS
2347
    closesocket(sockfd);
2348 2349
    WSACleanup();
#else
2350
    close(sockfd);
2351
#endif
2352

2353
    return 0;
2354 2355
}

2356
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
2357 2358 2359 2360 2361 2362
    char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
    if (NULL == dataBuf) {
        errorPrint("%s() LN%d, calloc failed! size:%d\n",
                __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
        return NULL;
    }
2363

2364 2365 2366
    int    dataLen = 0;
    dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
            "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);
2367

2368
    return dataBuf;
2369 2370
}

2371
static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) {
2372 2373 2374
    char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
    if (NULL == dataBuf) {
        printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
2375
        return NULL;
2376
    }
2377

2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390
    int    dataLen = 0;
    dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
    for (int i = 0; i < stbInfo->tagCount; i++) {
        if ((0 == strncasecmp(stbInfo->tags[i].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(stbInfo->tags[i].dataType,
                        "nchar", strlen("nchar")))) {
            if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
                printf("binary or nchar length overflow, max size:%u\n",
                        (uint32_t)TSDB_MAX_BINARY_LEN);
                tmfree(dataBuf);
                return NULL;
            }
2391

2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410
            int tagBufLen = stbInfo->tags[i].dataLen + 1;
            char* buf = (char*)calloc(tagBufLen, 1);
            if (NULL == buf) {
                printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
                tmfree(dataBuf);
                return NULL;
            }

            if (tableSeq % 2) {
                tstrncpy(buf, "beijing", tagBufLen);
            } else {
                tstrncpy(buf, "shanghai", tagBufLen);
            }
            //rand_string(buf, stbInfo->tags[i].dataLen);
            dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
                    "\'%s\', ", buf);
            tmfree(buf);
        } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                    "int", strlen("int"))) {
2411 2412 2413 2414 2415 2416 2417
            if ((g_args.demo_mode) && (i == 0)) {
                dataLen += snprintf(dataBuf + dataLen,
                        TSDB_MAX_SQL_LEN - dataLen,
                    "%d, ", tableSeq % 10);
            } else {
                dataLen += snprintf(dataBuf + dataLen,
                        TSDB_MAX_SQL_LEN - dataLen,
2418
                    "%d, ", tableSeq);
2419
            }
2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452
        } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                    "bigint", strlen("bigint"))) {
            dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
                    "%"PRId64", ", rand_bigint());
        }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                    "float", strlen("float"))) {
            dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
                    "%f, ", rand_float());
        }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                    "double", strlen("double"))) {
            dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
                    "%f, ", rand_double());
        }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                    "smallint", strlen("smallint"))) {
            dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
                    "%d, ", rand_smallint());
        }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                    "tinyint", strlen("tinyint"))) {
            dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
                    "%d, ", rand_tinyint());
        }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                    "bool", strlen("bool"))) {
            dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
                    "%d, ", rand_bool());
        }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                    "timestamp", strlen("timestamp"))) {
            dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
                    "%"PRId64", ", rand_bigint());
        }  else {
            printf("No support data type: %s\n", stbInfo->tags[i].dataType);
            tmfree(dataBuf);
            return NULL;
        }
2453
    }
2454

2455 2456 2457
    dataLen -= 2;
    dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");
    return dataBuf;
2458 2459
}

2460
static int calcRowLen(SSuperTable*  superTbls) {
2461 2462
  int colIndex;
  int  lenOfOneRow = 0;
2463

2464 2465
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2466

2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
2483
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2484
      lenOfOneRow += 42;
2485
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
2499

2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
2516
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2517 2518 2519 2520 2521 2522 2523 2524
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2525

2526 2527 2528 2529
  return 0;
}


2530
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
2531
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
2532
        int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) {
2533

2534
  char command[BUFFER_SIZE] = "\0";
2535 2536
  char limitBuf[100] = "\0";

2537
  TAOS_RES * res;
2538 2539 2540
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2541 2542

  if (offset >= 0) {
2543
    snprintf(limitBuf, 100, " limit %"PRId64" offset %"PRIu64"",
2544
            limit, offset);
2545 2546
  }

2547
  //get all child table name use cmd: select tbname from superTblName;
2548
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s",
2549
          dbName, sTblName, limitBuf);
2550

2551
  res = taos_query(taos, command);
2552 2553 2554 2555
  int32_t code = taos_errno(res);
  if (code != 0) {
    taos_free_result(res);
    taos_close(taos);
2556
    errorPrint("%s() LN%d, failed to run command %s\n",
2557
           __func__, __LINE__, command);
2558 2559 2560
    exit(-1);
  }

2561 2562
  int64_t childTblCount = (limit < 0)?10000:limit;
  int64_t count = 0;
2563 2564 2565
  if (childTblName == NULL) {
    childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (NULL ==  childTblName) {
2566 2567 2568 2569
    taos_free_result(res);
        taos_close(taos);
        errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__);
        exit(-1);
2570 2571 2572
    }
  }

2573
  char* pTblName = childTblName;
2574
  while((row = taos_fetch_row(res)) != NULL) {
2575
    int32_t* len = taos_fetch_lengths(res);
2576 2577 2578 2579 2580 2581 2582

    if (0 == strlen((char *)row[0])) {
        errorPrint("%s() LN%d, No.%"PRId64" table return empty name\n",
            __func__, __LINE__, count);
        exit(-1);
    }

2583
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2584 2585 2586
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2587
      char *tmp = realloc(childTblName,
2588
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2589 2590 2591
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2592
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
2593
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2594 2595
      } else {
        // exit, if allocate more memory failed
2596
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
2597
               __func__, __LINE__, dbName, sTblName);
2598 2599 2600 2601 2602 2603
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2604
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2605
  }
2606

2607 2608 2609 2610 2611 2612 2613
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2614
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
2615
        char* sTblName, char** childTblNameOfSuperTbl,
2616
        int64_t* childTblCountOfSuperTbl) {
2617

2618 2619
    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
2620
            -1, 0);
2621 2622
}

2623
static int getSuperTableFromServer(TAOS * taos, char* dbName,
2624
        SSuperTable*  superTbls) {
2625

2626
  char command[BUFFER_SIZE] = "\0";
2627
  TAOS_RES * res;
2628 2629
  TAOS_ROW row = NULL;
  int count = 0;
2630

2631
  //get schema use cmd: describe superTblName;
2632
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2633
  res = taos_query(taos, command);
2634 2635 2636
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2637
    taos_free_result(res);
2638 2639 2640 2641 2642 2643
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
2644
  while((row = taos_fetch_row(res)) != NULL) {
2645 2646 2647
    if (0 == count) {
      count++;
      continue;
2648
    }
2649 2650

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2651
      tstrncpy(superTbls->tags[tagIndex].field,
2652 2653
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
2654
      tstrncpy(superTbls->tags[tagIndex].dataType,
2655
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
2656
              min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes));
2657 2658 2659
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
2660 2661
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2662
      tagIndex++;
2663
    } else {
2664
      tstrncpy(superTbls->columns[columnIndex].field,
2665 2666
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
2667
      tstrncpy(superTbls->columns[columnIndex].dataType,
2668
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
2669
              min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes));
2670 2671 2672
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
2673 2674
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2686
/*
2687
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
2688
    //get all child table name use cmd: select tbname from superTblName;
2689 2690 2691
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2692
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2693 2694
      return -1;
    }
2695
    getAllChildNameOfSuperTable(taos, dbName,
2696 2697 2698
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2699
  }
2700
  */
2701 2702 2703
  return 0;
}

H
Haojun Liao 已提交
2704
static int createSuperTable(
2705 2706
        TAOS * taos, char* dbName,
        SSuperTable*  superTbl) {
H
Haojun Liao 已提交
2707

2708
    char command[BUFFER_SIZE] = "\0";
2709

2710 2711 2712
    char cols[STRING_LEN] = "\0";
    int colIndex;
    int len = 0;
2713

2714
    int  lenOfOneRow = 0;
2715

2716 2717 2718 2719 2720
    if (superTbl->columnCount == 0) {
        errorPrint("%s() LN%d, super table column count is %d\n",
                __func__, __LINE__, superTbl->columnCount);
        return -1;
    }
2721

2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766
    for (colIndex = 0; colIndex < superTbl->columnCount; colIndex++) {
        char* dataType = superTbl->columns[colIndex].dataType;

        if (strcasecmp(dataType, "BINARY") == 0) {
            len += snprintf(cols + len, STRING_LEN - len,
                    ", col%d %s(%d)", colIndex, "BINARY",
                    superTbl->columns[colIndex].dataLen);
            lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
        } else if (strcasecmp(dataType, "NCHAR") == 0) {
            len += snprintf(cols + len, STRING_LEN - len,
                    ", col%d %s(%d)", colIndex, "NCHAR",
                    superTbl->columns[colIndex].dataLen);
            lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
        } else if (strcasecmp(dataType, "INT") == 0)  {
            if ((g_args.demo_mode) && (colIndex == 1)) {
                    len += snprintf(cols + len, STRING_LEN - len,
                            ", VOLTAGE INT");
            } else {
                len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
            }
            lenOfOneRow += 11;
        } else if (strcasecmp(dataType, "BIGINT") == 0)  {
            len += snprintf(cols + len, STRING_LEN - len, ", col%d %s",
                    colIndex, "BIGINT");
            lenOfOneRow += 21;
        } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
            len += snprintf(cols + len, STRING_LEN - len, ", col%d %s",
                    colIndex, "SMALLINT");
            lenOfOneRow += 6;
        } else if (strcasecmp(dataType, "TINYINT") == 0)  {
            len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
            lenOfOneRow += 4;
        } else if (strcasecmp(dataType, "BOOL") == 0)  {
            len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
            lenOfOneRow += 6;
        } else if (strcasecmp(dataType, "FLOAT") == 0) {
            if (g_args.demo_mode) {
                if (colIndex == 0) {
                    len += snprintf(cols + len, STRING_LEN - len, ", CURRENT FLOAT");
                } else if (colIndex == 2) {
                    len += snprintf(cols + len, STRING_LEN - len, ", PHASE FLOAT");
                }
            } else {
                len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
            }
2767

2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782
            lenOfOneRow += 22;
        } else if (strcasecmp(dataType, "DOUBLE") == 0) {
            len += snprintf(cols + len, STRING_LEN - len, ", col%d %s",
                    colIndex, "DOUBLE");
            lenOfOneRow += 42;
        }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
            len += snprintf(cols + len, STRING_LEN - len, ", col%d %s",
                    colIndex, "TIMESTAMP");
            lenOfOneRow += 21;
        } else {
            taos_close(taos);
            errorPrint("%s() LN%d, config error data type : %s\n",
                    __func__, __LINE__, dataType);
            exit(-1);
        }
2783 2784
    }

2785
    superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp
2786

2787 2788 2789 2790 2791 2792 2793 2794
    // save for creating child table
    superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1);
    if (NULL == superTbl->colsOfCreateChildTable) {
        errorPrint("%s() LN%d, Failed when calloc, size:%d",
                __func__, __LINE__, len+1);
        taos_close(taos);
        exit(-1);
    }
2795

2796 2797 2798
    snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
    verbosePrint("%s() LN%d: %s\n",
            __func__, __LINE__, superTbl->colsOfCreateChildTable);
2799

2800 2801 2802 2803 2804
    if (superTbl->tagCount == 0) {
        errorPrint("%s() LN%d, super table tag count is %d\n",
                __func__, __LINE__, superTbl->tagCount);
        return -1;
    }
2805

2806 2807 2808
    char tags[STRING_LEN] = "\0";
    int tagIndex;
    len = 0;
2809

2810 2811 2812 2813 2814 2815
    int lenOfTagOfOneRow = 0;
    len += snprintf(tags + len, STRING_LEN - len, "(");
    for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) {
        char* dataType = superTbl->tags[tagIndex].dataType;

        if (strcasecmp(dataType, "BINARY") == 0) {
2816 2817 2818 2819 2820 2821 2822 2823
            if ((g_args.demo_mode) && (tagIndex == 1)) {
                len += snprintf(tags + len, STRING_LEN - len,
                        "loction BINARY(%d), ",
                        superTbl->tags[tagIndex].dataLen);
            } else {
                len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ",
                        tagIndex, "BINARY", superTbl->tags[tagIndex].dataLen);
            }
2824 2825 2826 2827 2828 2829
            lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
        } else if (strcasecmp(dataType, "NCHAR") == 0) {
            len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
                    "NCHAR", superTbl->tags[tagIndex].dataLen);
            lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
        } else if (strcasecmp(dataType, "INT") == 0)  {
2830 2831 2832 2833
            if ((g_args.demo_mode) && (tagIndex == 0)) {
                len += snprintf(tags + len, STRING_LEN - len, "groupId INT, ");
            } else {
                len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2834
                    "INT");
2835
            }
2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866
            lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11;
        } else if (strcasecmp(dataType, "BIGINT") == 0)  {
            len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                    "BIGINT");
            lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21;
        } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
            len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                    "SMALLINT");
            lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
        } else if (strcasecmp(dataType, "TINYINT") == 0)  {
            len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                    "TINYINT");
            lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4;
        } else if (strcasecmp(dataType, "BOOL") == 0)  {
            len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                    "BOOL");
            lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
        } else if (strcasecmp(dataType, "FLOAT") == 0) {
            len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                    "FLOAT");
            lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22;
        } else if (strcasecmp(dataType, "DOUBLE") == 0) {
            len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                    "DOUBLE");
            lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42;
        } else {
            taos_close(taos);
            errorPrint("%s() LN%d, config error tag type : %s\n",
                    __func__, __LINE__, dataType);
            exit(-1);
        }
2867
    }
2868

2869 2870
    len -= 2;
    len += snprintf(tags + len, STRING_LEN - len, ")");
2871

2872
    superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow;
2873

2874 2875 2876 2877 2878 2879 2880 2881 2882 2883
    snprintf(command, BUFFER_SIZE,
            "create table if not exists %s.%s (ts timestamp%s) tags %s",
            dbName, superTbl->sTblName, cols, tags);
    if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
        errorPrint( "create supertable %s failed!\n\n",
                superTbl->sTblName);
        return -1;
    }
    debugPrint("create supertable %s success!\n\n", superTbl->sTblName);
    return 0;
2884 2885
}

2886
static int createDatabasesAndStables() {
2887 2888 2889 2890 2891
    TAOS * taos = NULL;
    int    ret = 0;
    taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
    if (taos == NULL) {
        errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2892
        return -1;
2893 2894
    }
    char command[BUFFER_SIZE] = "\0";
2895

2896 2897 2898 2899 2900 2901 2902
    for (int i = 0; i < g_Dbs.dbCount; i++) {
        if (g_Dbs.db[i].drop) {
            sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
            if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
                taos_close(taos);
                return -1;
            }
2903

2904 2905 2906
            int dataLen = 0;
            dataLen += snprintf(command + dataLen,
                    BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2907

2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973
            if (g_Dbs.db[i].dbCfg.blocks > 0) {
                dataLen += snprintf(command + dataLen,
                        BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
            }
            if (g_Dbs.db[i].dbCfg.cache > 0) {
                dataLen += snprintf(command + dataLen,
                        BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
            }
            if (g_Dbs.db[i].dbCfg.days > 0) {
                dataLen += snprintf(command + dataLen,
                        BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
            }
            if (g_Dbs.db[i].dbCfg.keep > 0) {
                dataLen += snprintf(command + dataLen,
                        BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
            }
            if (g_Dbs.db[i].dbCfg.quorum > 1) {
                dataLen += snprintf(command + dataLen,
                        BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
            }
            if (g_Dbs.db[i].dbCfg.replica > 0) {
                dataLen += snprintf(command + dataLen,
                        BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
            }
            if (g_Dbs.db[i].dbCfg.update > 0) {
                dataLen += snprintf(command + dataLen,
                        BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
            }
            //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
            //  dataLen += snprintf(command + dataLen,
            //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
            //}
            if (g_Dbs.db[i].dbCfg.minRows > 0) {
                dataLen += snprintf(command + dataLen,
                        BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
            }
            if (g_Dbs.db[i].dbCfg.maxRows > 0) {
                dataLen += snprintf(command + dataLen,
                        BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
            }
            if (g_Dbs.db[i].dbCfg.comp > 0) {
                dataLen += snprintf(command + dataLen,
                        BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
            }
            if (g_Dbs.db[i].dbCfg.walLevel > 0) {
                dataLen += snprintf(command + dataLen,
                        BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
            }
            if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
                dataLen += snprintf(command + dataLen,
                        BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
            }
            if (g_Dbs.db[i].dbCfg.fsync > 0) {
                dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
                        " fsync %d", g_Dbs.db[i].dbCfg.fsync);
            }
            if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
#if NANO_SECOND_ENABLED == 1
                    || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                            "ns", strlen("ns")))
#endif
                    || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                            "us", strlen("us"))))  {
                dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
                        " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
            }
2974

2975 2976 2977 2978 2979 2980 2981
            if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
                taos_close(taos);
                errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
                return -1;
            }
            printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
        }
2982

2983 2984
        debugPrint("%s() LN%d supertbl count:%"PRIu64"\n",
                __func__, __LINE__, g_Dbs.db[i].superTblCount);
2985

2986
        int validStbCount = 0;
2987

2988 2989 2990 2991
        for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
            sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
                    g_Dbs.db[i].superTbls[j].sTblName);
            ret = queryDbExec(taos, command, NO_INSERT_TYPE, true);
2992

2993 2994 2995
            if ((ret != 0) || (g_Dbs.db[i].drop)) {
                ret = createSuperTable(taos, g_Dbs.db[i].dbName,
                        &g_Dbs.db[i].superTbls[j]);
2996

2997 2998 2999 3000 3001
                if (0 != ret) {
                    errorPrint("create super table %"PRIu64" failed!\n\n", j);
                    continue;
                }
            }
3002

3003 3004 3005 3006 3007 3008 3009
            ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
                    &g_Dbs.db[i].superTbls[j]);
            if (0 != ret) {
                errorPrint("\nget super table %s.%s info failed!\n\n",
                        g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName);
                continue;
            }
3010

3011 3012
            validStbCount ++;
        }
3013

3014 3015 3016 3017 3018
        g_Dbs.db[i].superTblCount = validStbCount;
    }

    taos_close(taos);
    return 0;
3019 3020
}

3021 3022
static void* createTable(void *sarg)
{
3023 3024
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
3025

S
Shuduo Sang 已提交
3026
  uint64_t  lastPrintTime = taosGetTimestampMs();
3027

3028
  int buff_len;
3029
  buff_len = BUFFER_SIZE / 8;
3030

3031 3032
  pThreadInfo->buffer = calloc(buff_len, 1);
  if (pThreadInfo->buffer == NULL) {
3033
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
3034 3035
    exit(-1);
  }
3036 3037 3038

  int len = 0;
  int batchNum = 0;
3039

3040
  verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n",
3041 3042
          __func__, __LINE__,
          pThreadInfo->start_table_from, pThreadInfo->end_table_to);
3043

3044
  for (uint64_t i = pThreadInfo->start_table_from;
3045
          i <= pThreadInfo->end_table_to; i++) {
3046
    if (0 == g_Dbs.use_metric) {
3047
      snprintf(pThreadInfo->buffer, buff_len,
3048
              "create table if not exists %s.%s%"PRIu64" %s;",
3049 3050 3051
              pThreadInfo->db_name,
              g_args.tb_prefix, i,
              pThreadInfo->cols);
3052
    } else {
3053 3054
      if (superTblInfo == NULL) {
        errorPrint("%s() LN%d, use metric, but super table info is NULL\n",
3055
                  __func__, __LINE__);
3056
        free(pThreadInfo->buffer);
3057 3058 3059 3060
        exit(-1);
      } else {
        if (0 == len) {
          batchNum = 0;
3061 3062
          memset(pThreadInfo->buffer, 0, buff_len);
          len += snprintf(pThreadInfo->buffer + len,
3063
                  buff_len - len, "create table ");
3064 3065 3066
        }
        char* tagsValBuf = NULL;
        if (0 == superTblInfo->tagSource) {
3067
          tagsValBuf = generateTagVaulesForStb(superTblInfo, i);
3068 3069
        } else {
          tagsValBuf = getTagValueFromTagSample(
3070 3071
                  superTblInfo,
                  i % superTblInfo->tagSampleCount);
3072 3073
        }
        if (NULL == tagsValBuf) {
3074
          free(pThreadInfo->buffer);
3075 3076
          return NULL;
        }
3077
        len += snprintf(pThreadInfo->buffer + len,
3078
                buff_len - len,
3079
                "if not exists %s.%s%"PRIu64" using %s.%s tags %s ",
3080 3081 3082
                pThreadInfo->db_name, superTblInfo->childTblPrefix,
                i, pThreadInfo->db_name,
                superTblInfo->sTblName, tagsValBuf);
3083 3084 3085
        free(tagsValBuf);
        batchNum++;
        if ((batchNum < superTblInfo->batchCreateTableNum)
3086 3087
                && ((buff_len - len)
                    >= (superTblInfo->lenOfTagOfOneRow + 256))) {
3088 3089
          continue;
        }
3090 3091 3092 3093
      }
    }

    len = 0;
3094 3095 3096 3097
    if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer,
                NO_INSERT_TYPE, false)){
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer);
      free(pThreadInfo->buffer);
3098 3099 3100
      return NULL;
    }

S
Shuduo Sang 已提交
3101
    uint64_t  currentPrintTime = taosGetTimestampMs();
3102
    if (currentPrintTime - lastPrintTime > 30*1000) {
3103
      printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n",
3104
              pThreadInfo->threadID, pThreadInfo->start_table_from, i);
3105 3106 3107
      lastPrintTime = currentPrintTime;
    }
  }
3108

3109
  if (0 != len) {
3110 3111 3112
    if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer,
                NO_INSERT_TYPE, false)) {
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer);
3113
    }
3114
  }
3115

3116
  free(pThreadInfo->buffer);
3117 3118 3119
  return NULL;
}

3120
static int startMultiThreadCreateChildTable(
3121
        char* cols, int threads, uint64_t tableFrom, int64_t ntables,
3122
        char* db_name, SSuperTable* superTblInfo) {
3123

3124
  pthread_t *pids = calloc(1, threads * sizeof(pthread_t));
3125
  threadInfo *infos = calloc(1, threads * sizeof(threadInfo));
3126 3127 3128 3129 3130 3131 3132 3133 3134 3135

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

S
Shuduo Sang 已提交
3136
  int64_t a = ntables / threads;
3137 3138 3139 3140 3141
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

S
Shuduo Sang 已提交
3142
  int64_t b = 0;
3143
  b = ntables % threads;
3144

3145 3146 3147
  for (int64_t i = 0; i < threads; i++) {
    threadInfo *pThreadInfo = infos + i;
    pThreadInfo->threadID = i;
3148
    tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN);
3149
    pThreadInfo->superTblInfo = superTblInfo;
3150
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
3151
    pThreadInfo->taos = taos_connect(
3152 3153 3154 3155 3156
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
3157
    if (pThreadInfo->taos == NULL) {
3158
      errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n",
3159
         __func__, __LINE__, taos_errstr(NULL));
3160
      free(pids);
3161
      free(infos);
3162 3163
      return -1;
    }
3164

3165
    pThreadInfo->start_table_from = tableFrom;
3166
    pThreadInfo->ntables = i<b?a+1:a;
3167 3168
    pThreadInfo->end_table_to = i < b ? tableFrom + a : tableFrom + a - 1;
    tableFrom = pThreadInfo->end_table_to + 1;
3169 3170 3171 3172
    pThreadInfo->use_metric = true;
    pThreadInfo->cols = cols;
    pThreadInfo->minDelay = UINT64_MAX;
    pthread_create(pids + i, NULL, createTable, pThreadInfo);
3173
  }
3174

3175 3176 3177 3178 3179
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
3180 3181
    threadInfo *pThreadInfo = infos + i;
    taos_close(pThreadInfo->taos);
3182 3183 3184
  }

  free(pids);
3185
  free(infos);
3186 3187

  return 0;
3188 3189
}

3190
static void createChildTables() {
3191 3192
    char tblColsBuf[MAX_SQL_SIZE];
    int len;
3193

3194 3195 3196 3197 3198
    for (int i = 0; i < g_Dbs.dbCount; i++) {
        if (g_Dbs.use_metric) {
            if (g_Dbs.db[i].superTblCount > 0) {
                // with super table
                for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
3199 3200 3201 3202
                    if ((AUTO_CREATE_SUBTBL
                                == g_Dbs.db[i].superTbls[j].autoCreateTable)
                            || (TBL_ALREADY_EXISTS
                                == g_Dbs.db[i].superTbls[j].childTblExists)) {
3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235
                        continue;
                    }
                    verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
                          g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
                    uint64_t startFrom = 0;
                    g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;

                    verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n",
                            __func__, __LINE__, g_totalChildTables, startFrom);

                    startMultiThreadCreateChildTable(
                            g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
                            g_Dbs.threadCountByCreateTbl,
                            startFrom,
                            g_Dbs.db[i].superTbls[j].childTblCount,
                            g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
                }
            }
        } else {
            // normal table
            len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
            for (int j = 0; j < g_args.num_of_CPR; j++) {
                if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
                        || (strncasecmp(g_args.datatype[j],
                                "NCHAR", strlen("NCHAR")) == 0)) {
                    snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                            ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary);
                } else {
                    snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                            ", COL%d %s", j, g_args.datatype[j]);
                }
                len = strlen(tblColsBuf);
            }
3236

3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248
            snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");

            verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n",
                    __func__, __LINE__,
                    g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
            startMultiThreadCreateChildTable(
                    tblColsBuf,
                    g_Dbs.threadCountByCreateTbl,
                    0,
                    g_args.num_of_tables,
                    g_Dbs.db[i].dbName,
                    NULL);
3249
        }
3250
    }
3251 3252 3253 3254 3255
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3256
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
3257 3258 3259
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
3260

3261 3262
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
3263
    printf("Failed to open tags file: %s, reason:%s\n",
3264
            superTblInfo->tagsFile, strerror(errno));
3265 3266 3267 3268 3269 3270 3271
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
3272

3273 3274 3275 3276 3277 3278 3279 3280 3281
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

3282
  while((readLen = tgetline(&line, &n, fp)) != -1) {
3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
3295
      char *tmp = realloc(tagDataBuf,
3296
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
3297 3298 3299
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
3300
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
3301
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3324
static int readSampleFromCsvFileToMem(
3325
        SSuperTable* superTblInfo) {
3326 3327 3328 3329
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
3330

3331 3332
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
3333 3334 3335
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
              superTblInfo->sampleFile, strerror(errno));
      return -1;
3336
  }
3337

3338
  assert(superTblInfo->sampleDataBuf);
3339
  memset(superTblInfo->sampleDataBuf, 0,
3340
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
3341
  while(1) {
3342
    readLen = tgetline(&line, &n, fp);
3343 3344
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
3345
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
3346
                superTblInfo->sampleFile, strerror(errno));
3347
        fclose(fp);
3348 3349 3350 3351
        return -1;
      }
      continue;
    }
3352

3353 3354 3355 3356 3357 3358 3359 3360 3361
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
3362
      printf("sample row len[%d] overflow define schema len[%"PRIu64"], so discard this row\n",
3363
              (int32_t)readLen, superTblInfo->lenOfOneRow);
3364 3365 3366
      continue;
    }

3367
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
3368
          line, readLen);
3369 3370 3371 3372 3373 3374 3375
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

3376
  fclose(fp);
3377 3378 3379 3380
  tmfree(line);
  return 0;
}

3381
static bool getColumnAndTagTypeFromInsertJsonFile(
3382
        cJSON* stbInfo, SSuperTable* superTbls) {
3383
  bool  ret = false;
3384

3385
  // columns
3386 3387
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
3388
    printf("ERROR: failed to read json, columns not found\n");
3389 3390 3391 3392 3393 3394
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
3395

3396
  int columnSize = cJSON_GetArraySize(columns);
3397
  if ((columnSize + 1/* ts */) > TSDB_MAX_COLUMNS) {
3398
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
3399
            __func__, __LINE__, TSDB_MAX_COLUMNS);
3400 3401 3402 3403 3404 3405
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
3406

3407
  //superTbls->columnCount = columnSize;
3408 3409 3410 3411 3412 3413 3414
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
3415
      count = countObj->valueint;
3416
    } else if (countObj && countObj->type != cJSON_Number) {
3417
      errorPrint("%s() LN%d, failed to read json, column count not found\n",
3418
          __func__, __LINE__);
3419 3420 3421 3422 3423
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3424
    // column info
3425 3426
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
3427 3428 3429
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d: failed to read json, column type not found\n",
3430
          __func__, __LINE__);
3431 3432
      goto PARSE_OVER;
    }
3433
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3434
    tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1);
3435

3436 3437
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3438
      columnCase.dataLen = dataLen->valueint;
3439
    } else if (dataLen && dataLen->type != cJSON_Number) {
3440
      debugPrint("%s() LN%d: failed to read json, column len not found\n",
3441
          __func__, __LINE__);
3442 3443 3444 3445
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
3446

3447
    for (int n = 0; n < count; ++n) {
3448
      tstrncpy(superTbls->columns[index].dataType,
3449
              columnCase.dataType, strlen(columnCase.dataType) + 1);
3450
      superTbls->columns[index].dataLen = columnCase.dataLen;
3451 3452
      index++;
    }
3453
  }
3454

3455
  if ((index + 1 /* ts */) > MAX_NUM_COLUMNS) {
3456
    errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n",
3457
            __func__, __LINE__, MAX_NUM_COLUMNS);
3458 3459 3460
    goto PARSE_OVER;
  }

3461
  superTbls->columnCount = index;
3462

3463 3464
  count = 1;
  index = 0;
3465
  // tags
3466 3467
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
3468
    errorPrint("%s() LN%d, failed to read json, tags not found\n",
3469
        __func__, __LINE__);
3470 3471
    goto PARSE_OVER;
  }
3472

3473
  int tagSize = cJSON_GetArraySize(tags);
3474
  if (tagSize > TSDB_MAX_TAGS) {
3475
    errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n",
3476
        __func__, __LINE__, TSDB_MAX_TAGS);
3477 3478
    goto PARSE_OVER;
  }
3479 3480

  //superTbls->tagCount = tagSize;
3481 3482 3483
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
3484

3485 3486 3487
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
3488
      count = countObj->valueint;
3489
    } else if (countObj && countObj->type != cJSON_Number) {
3490
      printf("ERROR: failed to read json, column count not found\n");
3491 3492 3493 3494 3495
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3496
    // column info
3497 3498
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
3499 3500 3501
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d, failed to read json, tag type not found\n",
3502
          __func__, __LINE__);
3503 3504
      goto PARSE_OVER;
    }
3505
    tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1);
3506

3507 3508
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3509
      columnCase.dataLen = dataLen->valueint;
3510
    } else if (dataLen && dataLen->type != cJSON_Number) {
3511
      errorPrint("%s() LN%d, failed to read json, column len not found\n",
3512
          __func__, __LINE__);
3513 3514 3515
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
3516 3517
    }

3518
    for (int n = 0; n < count; ++n) {
3519
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType,
3520
          strlen(columnCase.dataType) + 1);
3521
      superTbls->tags[index].dataLen = columnCase.dataLen;
3522 3523
      index++;
    }
3524
  }
3525

3526
  if (index > TSDB_MAX_TAGS) {
3527
    errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n",
3528
        __func__, __LINE__, TSDB_MAX_TAGS);
3529 3530 3531
    goto PARSE_OVER;
  }

3532 3533
  superTbls->tagCount = index;

3534
  if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > TSDB_MAX_COLUMNS) {
3535
    errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n",
3536
        __func__, __LINE__, TSDB_MAX_COLUMNS);
3537 3538
    goto PARSE_OVER;
  }
3539 3540
  ret = true;

3541
PARSE_OVER:
3542 3543 3544 3545
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
3546
    bool  ret = false;
3547

3548 3549 3550
    cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
    if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
        tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3551
    }
3552

3553 3554 3555 3556 3557
    cJSON* host = cJSON_GetObjectItem(root, "host");
    if (host && host->type == cJSON_String && host->valuestring != NULL) {
        tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE);
    } else if (!host) {
        tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3558
    } else {
3559 3560
        printf("ERROR: failed to read json, host not found\n");
        goto PARSE_OVER;
3561 3562
    }

3563 3564 3565 3566 3567
    cJSON* port = cJSON_GetObjectItem(root, "port");
    if (port && port->type == cJSON_Number) {
        g_Dbs.port = port->valueint;
    } else if (!port) {
        g_Dbs.port = 6030;
3568
    }
3569

3570 3571 3572 3573 3574
    cJSON* user = cJSON_GetObjectItem(root, "user");
    if (user && user->type == cJSON_String && user->valuestring != NULL) {
        tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE);
    } else if (!user) {
        tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE);
3575 3576
    }

3577 3578 3579 3580 3581
    cJSON* password = cJSON_GetObjectItem(root, "password");
    if (password && password->type == cJSON_String && password->valuestring != NULL) {
        tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE);
    } else if (!password) {
        tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE);
3582 3583
    }

3584 3585 3586 3587 3588
    cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
    if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
        tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
    } else if (!resultfile) {
        tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
3589 3590
    }

3591 3592 3593 3594 3595
    cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
    if (threads && threads->type == cJSON_Number) {
        g_Dbs.threadCount = threads->valueint;
    } else if (!threads) {
        g_Dbs.threadCount = 1;
3596
    } else {
3597 3598
        printf("ERROR: failed to read json, threads not found\n");
        goto PARSE_OVER;
3599
    }
3600

3601 3602 3603 3604 3605
    cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
    if (threads2 && threads2->type == cJSON_Number) {
        g_Dbs.threadCountByCreateTbl = threads2->valueint;
    } else if (!threads2) {
        g_Dbs.threadCountByCreateTbl = 1;
3606
    } else {
3607 3608 3609
        errorPrint("%s() LN%d, failed to read json, threads2 not found\n",
                __func__, __LINE__);
        goto PARSE_OVER;
3610
    }
3611

3612 3613 3614 3615 3616 3617 3618 3619 3620 3621
    cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
    if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
        if (gInsertInterval->valueint <0) {
            errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n",
                    __func__, __LINE__);
            goto PARSE_OVER;
        }
        g_args.insert_interval = gInsertInterval->valueint;
    } else if (!gInsertInterval) {
        g_args.insert_interval = 0;
3622
    } else {
3623 3624 3625
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
        goto PARSE_OVER;
3626
    }
3627

3628 3629 3630 3631 3632 3633
    cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows");
    if (interlaceRows && interlaceRows->type == cJSON_Number) {
        if (interlaceRows->valueint < 0) {
            errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
                    __func__, __LINE__);
            goto PARSE_OVER;
3634

3635 3636 3637 3638
        }
        g_args.interlace_rows = interlaceRows->valueint;
    } else if (!interlaceRows) {
        g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3639
    } else {
3640 3641 3642
        errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
                __func__, __LINE__);
        goto PARSE_OVER;
3643 3644
    }

3645 3646 3647 3648 3649 3650 3651 3652 3653 3654
    cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
    if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
        if (maxSqlLen->valueint < 0) {
            errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
                    __func__, __LINE__);
            goto PARSE_OVER;
        }
        g_args.max_sql_len = maxSqlLen->valueint;
    } else if (!maxSqlLen) {
        g_args.max_sql_len = (1024*1024);
3655
    } else {
3656 3657 3658
        errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
                __func__, __LINE__);
        goto PARSE_OVER;
3659 3660
    }

3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677
    cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
    if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
        if (numRecPerReq->valueint <= 0) {
            errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n",
                    __func__, __LINE__);
            goto PARSE_OVER;
        } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) {
            printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n",
                    numRecPerReq->valueint, MAX_RECORDS_PER_REQ);
            printf("        number of records per request value will be set to %d\n\n",
                    MAX_RECORDS_PER_REQ);
            prompt();
            numRecPerReq->valueint = MAX_RECORDS_PER_REQ;
        }
        g_args.num_of_RPR = numRecPerReq->valueint;
    } else if (!numRecPerReq) {
        g_args.num_of_RPR = MAX_RECORDS_PER_REQ;
3678
    } else {
3679 3680 3681
        errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n",
                __func__, __LINE__);
        goto PARSE_OVER;
3682 3683
    }

3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696
    cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
    if (answerPrompt
            && answerPrompt->type == cJSON_String
            && answerPrompt->valuestring != NULL) {
        if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
            g_args.answer_yes = false;
        } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
            g_args.answer_yes = true;
        } else {
            g_args.answer_yes = false;
        }
    } else if (!answerPrompt) {
        g_args.answer_yes = true;   // default is no, mean answer_yes.
3697
    } else {
3698 3699
        errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n");
        goto PARSE_OVER;
3700 3701
    }

3702 3703 3704 3705 3706 3707 3708 3709
    // rows per table need be less than insert batch
    if (g_args.interlace_rows > g_args.num_of_RPR) {
        printf("NOTICE: interlace rows value %u > num_of_records_per_req %u\n\n",
                g_args.interlace_rows, g_args.num_of_RPR);
        printf("        interlace rows value will be set to num_of_records_per_req %u\n\n",
                g_args.num_of_RPR);
        prompt();
        g_args.interlace_rows = g_args.num_of_RPR;
3710 3711
    }

3712 3713 3714 3715
    cJSON* dbs = cJSON_GetObjectItem(root, "databases");
    if (!dbs || dbs->type != cJSON_Array) {
        printf("ERROR: failed to read json, databases not found\n");
        goto PARSE_OVER;
3716
    }
3717

3718 3719 3720 3721 3722 3723
    int dbSize = cJSON_GetArraySize(dbs);
    if (dbSize > MAX_DB_COUNT) {
        errorPrint(
                "ERROR: failed to read json, databases size overflow, max database is %d\n",
                MAX_DB_COUNT);
        goto PARSE_OVER;
3724 3725
    }

3726 3727 3728 3729
    g_Dbs.dbCount = dbSize;
    for (int i = 0; i < dbSize; ++i) {
        cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
        if (dbinfos == NULL) continue;
3730

3731 3732 3733 3734 3735 3736
        // dbinfo
        cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
        if (!dbinfo || dbinfo->type != cJSON_Object) {
            printf("ERROR: failed to read json, dbinfo not found\n");
            goto PARSE_OVER;
        }
3737

3738 3739 3740 3741 3742 3743
        cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
        if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
            printf("ERROR: failed to read json, db name not found\n");
            goto PARSE_OVER;
        }
        tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, TSDB_DB_NAME_LEN);
3744

3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758
        cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
        if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
            if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) {
                g_Dbs.db[i].drop = true;
            } else {
                g_Dbs.db[i].drop = false;
            }
        } else if (!drop) {
            g_Dbs.db[i].drop = g_args.drop_database;
        } else {
            errorPrint("%s() LN%d, failed to read json, drop input mistake\n",
                    __func__, __LINE__);
            goto PARSE_OVER;
        }
3759

3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770
        cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
        if (precision && precision->type == cJSON_String
                && precision->valuestring != NULL) {
            tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
                    8);
        } else if (!precision) {
            memset(g_Dbs.db[i].dbCfg.precision, 0, 8);
        } else {
            printf("ERROR: failed to read json, precision not found\n");
            goto PARSE_OVER;
        }
3771

3772 3773 3774 3775 3776
        cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
        if (update && update->type == cJSON_Number) {
            g_Dbs.db[i].dbCfg.update = update->valueint;
        } else if (!update) {
            g_Dbs.db[i].dbCfg.update = -1;
3777
        } else {
3778 3779
            printf("ERROR: failed to read json, update not found\n");
            goto PARSE_OVER;
3780
        }
3781

3782 3783 3784 3785 3786 3787 3788 3789 3790
        cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
        if (replica && replica->type == cJSON_Number) {
            g_Dbs.db[i].dbCfg.replica = replica->valueint;
        } else if (!replica) {
            g_Dbs.db[i].dbCfg.replica = -1;
        } else {
            printf("ERROR: failed to read json, replica not found\n");
            goto PARSE_OVER;
        }
3791

3792 3793 3794 3795 3796 3797 3798 3799 3800
        cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
        if (keep && keep->type == cJSON_Number) {
            g_Dbs.db[i].dbCfg.keep = keep->valueint;
        } else if (!keep) {
            g_Dbs.db[i].dbCfg.keep = -1;
        } else {
            printf("ERROR: failed to read json, keep not found\n");
            goto PARSE_OVER;
        }
3801

3802 3803 3804 3805 3806
        cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
        if (days && days->type == cJSON_Number) {
            g_Dbs.db[i].dbCfg.days = days->valueint;
        } else if (!days) {
            g_Dbs.db[i].dbCfg.days = -1;
3807
        } else {
3808
            printf("ERROR: failed to read json, days not found\n");
3809 3810
            goto PARSE_OVER;
        }
3811

3812 3813 3814 3815 3816 3817 3818
        cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
        if (cache && cache->type == cJSON_Number) {
            g_Dbs.db[i].dbCfg.cache = cache->valueint;
        } else if (!cache) {
            g_Dbs.db[i].dbCfg.cache = -1;
        } else {
            printf("ERROR: failed to read json, cache not found\n");
3819
            goto PARSE_OVER;
3820 3821
        }

3822 3823 3824 3825 3826 3827 3828
        cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
        if (blocks && blocks->type == cJSON_Number) {
            g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
        } else if (!blocks) {
            g_Dbs.db[i].dbCfg.blocks = -1;
        } else {
            printf("ERROR: failed to read json, block not found\n");
3829
            goto PARSE_OVER;
3830 3831
        }

3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850
        //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
        //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
        //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
        //} else if (!maxtablesPerVnode) {
        //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
        //} else {
        // printf("failed to read json, maxtablesPerVnode not found");
        // goto PARSE_OVER;
        //}

        cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
        if (minRows && minRows->type == cJSON_Number) {
            g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
        } else if (!minRows) {
            g_Dbs.db[i].dbCfg.minRows = 0;    // 0 means default
        } else {
            printf("ERROR: failed to read json, minRows not found\n");
            goto PARSE_OVER;
        }
3851

3852 3853 3854 3855 3856 3857 3858 3859 3860
        cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
        if (maxRows && maxRows->type == cJSON_Number) {
            g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
        } else if (!maxRows) {
            g_Dbs.db[i].dbCfg.maxRows = 0;    // 0 means default
        } else {
            printf("ERROR: failed to read json, maxRows not found\n");
            goto PARSE_OVER;
        }
3861

3862 3863 3864 3865 3866 3867 3868 3869 3870
        cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
        if (comp && comp->type == cJSON_Number) {
            g_Dbs.db[i].dbCfg.comp = comp->valueint;
        } else if (!comp) {
            g_Dbs.db[i].dbCfg.comp = -1;
        } else {
            printf("ERROR: failed to read json, comp not found\n");
            goto PARSE_OVER;
        }
3871

3872 3873 3874 3875 3876 3877 3878 3879 3880
        cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
        if (walLevel && walLevel->type == cJSON_Number) {
            g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
        } else if (!walLevel) {
            g_Dbs.db[i].dbCfg.walLevel = -1;
        } else {
            printf("ERROR: failed to read json, walLevel not found\n");
            goto PARSE_OVER;
        }
3881

3882 3883 3884 3885 3886
        cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
        if (cacheLast && cacheLast->type == cJSON_Number) {
            g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
        } else if (!cacheLast) {
            g_Dbs.db[i].dbCfg.cacheLast = -1;
3887
        } else {
3888 3889
            printf("ERROR: failed to read json, cacheLast not found\n");
            goto PARSE_OVER;
3890
        }
3891

3892 3893 3894 3895 3896
        cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
        if (quorum && quorum->type == cJSON_Number) {
            g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
        } else if (!quorum) {
            g_Dbs.db[i].dbCfg.quorum = 1;
3897
        } else {
3898 3899
            printf("failed to read json, quorum input mistake");
            goto PARSE_OVER;
3900
        }
3901 3902 3903 3904 3905 3906 3907 3908 3909 3910

        cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
        if (fsync && fsync->type == cJSON_Number) {
            g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
        } else if (!fsync) {
            g_Dbs.db[i].dbCfg.fsync = -1;
        } else {
            errorPrint("%s() LN%d, failed to read json, fsync input mistake\n",
                    __func__, __LINE__);
            goto PARSE_OVER;
3911 3912
        }

3913 3914 3915 3916 3917 3918
        // super_talbes
        cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
        if (!stables || stables->type != cJSON_Array) {
            errorPrint("%s() LN%d, failed to read json, super_tables not found\n",
                    __func__, __LINE__);
            goto PARSE_OVER;
3919
        }
3920

3921 3922 3923 3924 3925 3926
        int stbSize = cJSON_GetArraySize(stables);
        if (stbSize > MAX_SUPER_TABLE_COUNT) {
            errorPrint(
                    "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n",
                    __func__, __LINE__, MAX_SUPER_TABLE_COUNT);
            goto PARSE_OVER;
3927
        }
3928

3929 3930 3931 3932
        g_Dbs.db[i].superTblCount = stbSize;
        for (int j = 0; j < stbSize; ++j) {
            cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
            if (stbInfo == NULL) continue;
3933

3934 3935 3936 3937 3938 3939 3940 3941 3942 3943
            // dbinfo
            cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
            if (!stbName || stbName->type != cJSON_String
                    || stbName->valuestring == NULL) {
                errorPrint("%s() LN%d, failed to read json, stb name not found\n",
                        __func__, __LINE__);
                goto PARSE_OVER;
            }
            tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring,
                    TSDB_TABLE_NAME_LEN);
3944

3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970
            cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
            if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
                printf("ERROR: failed to read json, childtable_prefix not found\n");
                goto PARSE_OVER;
            }
            tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring,
                    TSDB_TABLE_NAME_LEN - 20);

            cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table");
            if (autoCreateTbl
                    && autoCreateTbl->type == cJSON_String
                    && autoCreateTbl->valuestring != NULL) {
                if ((0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3))
                        && (TBL_ALREADY_EXISTS != g_Dbs.db[i].superTbls[j].childTblExists)) {
                    g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
                } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
                    g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
                } else {
                    g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
                }
            } else if (!autoCreateTbl) {
                g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
            } else {
                printf("ERROR: failed to read json, auto_create_table not found\n");
                goto PARSE_OVER;
            }
3971

3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259
            cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
            if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
                g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
            } else if (!batchCreateTbl) {
                g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
            } else {
                printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
                goto PARSE_OVER;
            }

            cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
            if (childTblExists
                    && childTblExists->type == cJSON_String
                    && childTblExists->valuestring != NULL) {
                if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3))
                        && (g_Dbs.db[i].drop == false)) {
                    g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
                } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2)
                            || (g_Dbs.db[i].drop == true))) {
                    g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
                } else {
                    g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
                }
            } else if (!childTblExists) {
                g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
            } else {
                errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n",
                        __func__, __LINE__);
                goto PARSE_OVER;
            }

            if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
                g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
            }

            cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
            if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
                errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n",
                        __func__, __LINE__);
                goto PARSE_OVER;
            }
            g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

            cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
            if (dataSource && dataSource->type == cJSON_String
                    && dataSource->valuestring != NULL) {
                tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
                        dataSource->valuestring, TSDB_DB_NAME_LEN);
            } else if (!dataSource) {
                tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", TSDB_DB_NAME_LEN);
            } else {
                errorPrint("%s() LN%d, failed to read json, data_source not found\n",
                        __func__, __LINE__);
                goto PARSE_OVER;
            }

            cJSON *stbIface = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt
            if (stbIface && stbIface->type == cJSON_String
                    && stbIface->valuestring != NULL) {
                if (0 == strcasecmp(stbIface->valuestring, "taosc")) {
                    g_Dbs.db[i].superTbls[j].iface= TAOSC_IFACE;
                } else if (0 == strcasecmp(stbIface->valuestring, "rest")) {
                    g_Dbs.db[i].superTbls[j].iface= REST_IFACE;
#if STMT_IFACE_ENABLED == 1
                } else if (0 == strcasecmp(stbIface->valuestring, "stmt")) {
                    g_Dbs.db[i].superTbls[j].iface= STMT_IFACE;
#endif
                } else {
                    errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n",
                            __func__, __LINE__, stbIface->valuestring);
                    goto PARSE_OVER;
                }
            } else if (!stbIface) {
                g_Dbs.db[i].superTbls[j].iface = TAOSC_IFACE;
            } else {
                errorPrint("%s", "failed to read json, insert_mode not found\n");
                goto PARSE_OVER;
            }

            cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
            if ((childTbl_limit) && (g_Dbs.db[i].drop != true)
                    && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
                if (childTbl_limit->type != cJSON_Number) {
                    printf("ERROR: failed to read json, childtable_limit\n");
                    goto PARSE_OVER;
                }
                g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
            } else {
                g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid.
            }

            cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
            if ((childTbl_offset) && (g_Dbs.db[i].drop != true)
                    && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
                if ((childTbl_offset->type != cJSON_Number)
                        || (0 > childTbl_offset->valueint)) {
                    printf("ERROR: failed to read json, childtable_offset\n");
                    goto PARSE_OVER;
                }
                g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
            } else {
                g_Dbs.db[i].superTbls[j].childTblOffset = 0;
            }

            cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
            if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
                tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                        ts->valuestring, TSDB_DB_NAME_LEN);
            } else if (!ts) {
                tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                        "now", TSDB_DB_NAME_LEN);
            } else {
                printf("ERROR: failed to read json, start_timestamp not found\n");
                goto PARSE_OVER;
            }

            cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
            if (timestampStep && timestampStep->type == cJSON_Number) {
                g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
            } else if (!timestampStep) {
                g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
            } else {
                printf("ERROR: failed to read json, timestamp_step not found\n");
                goto PARSE_OVER;
            }

            cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
            if (sampleFormat && sampleFormat->type
                    == cJSON_String && sampleFormat->valuestring != NULL) {
                tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
                        sampleFormat->valuestring, TSDB_DB_NAME_LEN);
            } else if (!sampleFormat) {
                tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", TSDB_DB_NAME_LEN);
            } else {
                printf("ERROR: failed to read json, sample_format not found\n");
                goto PARSE_OVER;
            }

            cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
            if (sampleFile && sampleFile->type == cJSON_String
                    && sampleFile->valuestring != NULL) {
                tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
                        sampleFile->valuestring, MAX_FILE_NAME_LEN);
            } else if (!sampleFile) {
                memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
            } else {
                printf("ERROR: failed to read json, sample_file not found\n");
                goto PARSE_OVER;
            }

            cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
            if ((tagsFile && tagsFile->type == cJSON_String)
                    && (tagsFile->valuestring != NULL)) {
                tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
                        tagsFile->valuestring, MAX_FILE_NAME_LEN);
                if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
                    g_Dbs.db[i].superTbls[j].tagSource = 0;
                } else {
                    g_Dbs.db[i].superTbls[j].tagSource = 1;
                }
            } else if (!tagsFile) {
                memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
                g_Dbs.db[i].superTbls[j].tagSource = 0;
            } else {
                printf("ERROR: failed to read json, tags_file not found\n");
                goto PARSE_OVER;
            }

            cJSON* stbMaxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
            if (stbMaxSqlLen && stbMaxSqlLen->type == cJSON_Number) {
                int32_t len = stbMaxSqlLen->valueint;
                if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
                    len = TSDB_MAX_ALLOWED_SQL_LEN;
                } else if (len < 5) {
                    len = 5;
                }
                g_Dbs.db[i].superTbls[j].maxSqlLen = len;
            } else if (!maxSqlLen) {
                g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len;
            } else {
                errorPrint("%s() LN%d, failed to read json, stbMaxSqlLen input mistake\n",
                        __func__, __LINE__);
                goto PARSE_OVER;
            }
            /*
               cJSON *multiThreadWriteOneTbl =
               cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
               if (multiThreadWriteOneTbl
               && multiThreadWriteOneTbl->type == cJSON_String
               && multiThreadWriteOneTbl->valuestring != NULL) {
               if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
               g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
               } else {
               g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
               }
               } else if (!multiThreadWriteOneTbl) {
               g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
               } else {
               printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
               goto PARSE_OVER;
               }
               */
            cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
            if (insertRows && insertRows->type == cJSON_Number) {
                if (insertRows->valueint < 0) {
                    errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                            __func__, __LINE__);
                    goto PARSE_OVER;
                }
                g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
            } else if (!insertRows) {
                g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
            } else {
                errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                        __func__, __LINE__);
                goto PARSE_OVER;
            }

            cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows");
            if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) {
                if (stbInterlaceRows->valueint < 0) {
                    errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n",
                            __func__, __LINE__);
                    goto PARSE_OVER;
                }
                g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint;

                if (g_Dbs.db[i].superTbls[j].interlaceRows > g_Dbs.db[i].superTbls[j].insertRows) {
                    printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %u > insert_rows %"PRId64"\n\n",
                            i, j, g_Dbs.db[i].superTbls[j].interlaceRows,
                            g_Dbs.db[i].superTbls[j].insertRows);
                    printf("        interlace rows value will be set to insert_rows %"PRId64"\n\n",
                            g_Dbs.db[i].superTbls[j].insertRows);
                    prompt();
                    g_Dbs.db[i].superTbls[j].interlaceRows = g_Dbs.db[i].superTbls[j].insertRows;
                }
            } else if (!stbInterlaceRows) {
                g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
            } else {
                errorPrint(
                        "%s() LN%d, failed to read json, interlace rows input mistake\n",
                        __func__, __LINE__);
                goto PARSE_OVER;
            }

            cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
            if (disorderRatio && disorderRatio->type == cJSON_Number) {
                if (disorderRatio->valueint > 50)
                    disorderRatio->valueint = 50;

                if (disorderRatio->valueint < 0)
                    disorderRatio->valueint = 0;

                g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
            } else if (!disorderRatio) {
                g_Dbs.db[i].superTbls[j].disorderRatio = 0;
            } else {
                printf("ERROR: failed to read json, disorderRatio not found\n");
                goto PARSE_OVER;
            }

            cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
            if (disorderRange && disorderRange->type == cJSON_Number) {
                g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
            } else if (!disorderRange) {
                g_Dbs.db[i].superTbls[j].disorderRange = 1000;
            } else {
                printf("ERROR: failed to read json, disorderRange not found\n");
                goto PARSE_OVER;
            }

            cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
            if (insertInterval && insertInterval->type == cJSON_Number) {
                g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
                if (insertInterval->valueint < 0) {
                    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                            __func__, __LINE__);
                    goto PARSE_OVER;
                }
            } else if (!insertInterval) {
                verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n",
                        __func__, __LINE__, g_args.insert_interval);
                g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
            } else {
                errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                        __func__, __LINE__);
                goto PARSE_OVER;
            }
4260

4261 4262 4263 4264 4265
            int retVal = getColumnAndTagTypeFromInsertJsonFile(
                    stbInfo, &g_Dbs.db[i].superTbls[j]);
            if (false == retVal) {
                goto PARSE_OVER;
            }
4266
        }
4267
    }
4268

4269
    ret = true;
4270

4271
PARSE_OVER:
4272
    return ret;
4273 4274 4275 4276 4277 4278 4279
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
4280
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
4281 4282 4283 4284
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
4285
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE);
4286
  } else if (!host) {
4287
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
4288
  } else {
4289
    printf("ERROR: failed to read json, host not found\n");
4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
4302
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE);
4303
  } else if (!user) {
4304
    tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ;
4305 4306 4307 4308
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
4309
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE);
4310
  } else if (!password) {
4311
    tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);;
4312
  }
H
hzcheng 已提交
4313

4314
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
4315
  if (answerPrompt && answerPrompt->type == cJSON_String
4316
          && answerPrompt->valuestring != NULL) {
4317 4318 4319 4320
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
4321
    } else {
4322
      g_args.answer_yes = false;
H
hzcheng 已提交
4323
    }
4324 4325 4326
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
4327
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
4328
    goto PARSE_OVER;
4329
  }
4330

4331 4332
  cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
  if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
4333
    if (gQueryTimes->valueint <= 0) {
4334 4335
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4336 4337
      goto PARSE_OVER;
    }
4338 4339 4340 4341
    g_args.query_times = gQueryTimes->valueint;
  } else if (!gQueryTimes) {
    g_args.query_times = 1;
  } else {
4342
    errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4343
        __func__, __LINE__);
4344 4345 4346
    goto PARSE_OVER;
  }

4347 4348
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
4349
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, TSDB_DB_NAME_LEN);
4350
  } else if (!dbs) {
4351
    printf("ERROR: failed to read json, databases not found\n");
4352
    goto PARSE_OVER;
H
hzcheng 已提交
4353
  }
4354 4355 4356

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
4357
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
4358
  } else if (!queryMode) {
4359
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
4360
  } else {
4361
    printf("ERROR: failed to read json, query_mode not found\n");
4362
    goto PARSE_OVER;
H
hzcheng 已提交
4363
  }
4364

4365
  // specified_table_query
4366 4367
  cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!specifiedQuery) {
4368
    g_queryInfo.specifiedQueryInfo.concurrent = 1;
4369 4370
    g_queryInfo.specifiedQueryInfo.sqlCount = 0;
  } else if (specifiedQuery->type != cJSON_Object) {
4371
    printf("ERROR: failed to read json, super_table_query not found\n");
4372
    goto PARSE_OVER;
4373
  } else {
4374 4375 4376 4377 4378
    cJSON* queryInterval = cJSON_GetObjectItem(specifiedQuery, "query_interval");
    if (queryInterval && queryInterval->type == cJSON_Number) {
      g_queryInfo.specifiedQueryInfo.queryInterval = queryInterval->valueint;
    } else if (!queryInterval) {
      g_queryInfo.specifiedQueryInfo.queryInterval = 0;
4379 4380
    }

4381
    cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery,
4382
        "query_times");
4383
    if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
4384
      if (specifiedQueryTimes->valueint <= 0) {
4385 4386
        errorPrint(
                "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4387
          __func__, __LINE__, specifiedQueryTimes->valueint);
4388 4389 4390
        goto PARSE_OVER;

      }
4391 4392 4393 4394
      g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint;
    } else if (!specifiedQueryTimes) {
      g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times;
    } else {
4395
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4396
          __func__, __LINE__);
4397
      goto PARSE_OVER;
4398
    }
4399

4400
    cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent");
4401
    if (concurrent && concurrent->type == cJSON_Number) {
4402
      if (concurrent->valueint <= 0) {
4403
        errorPrint(
4404
                "%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n",
4405 4406
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount,
4407
              g_queryInfo.specifiedQueryInfo.concurrent);
4408 4409
        goto PARSE_OVER;
      }
4410
      g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint;
4411
    } else if (!concurrent) {
4412
      g_queryInfo.specifiedQueryInfo.concurrent = 1;
4413
    }
4414

4415 4416 4417 4418 4419 4420 4421
    cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode");
    if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String
        && specifiedAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE;
4422
      } else {
4423
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4424
            __func__, __LINE__);
4425 4426 4427
        goto PARSE_OVER;
      }
    } else {
4428
      g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
4429
    }
4430

4431
    cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval");
4432
    if (interval && interval->type == cJSON_Number) {
4433
      g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint;
4434
    } else if (!interval) {
4435 4436
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4437
      g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000;
4438
    }
4439

4440
    cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart");
4441
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
4442
      if (0 == strcmp("yes", restart->valuestring)) {
4443
        g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4444
      } else if (0 == strcmp("no", restart->valuestring)) {
4445
        g_queryInfo.specifiedQueryInfo.subscribeRestart = false;
4446
      } else {
4447
        printf("ERROR: failed to read json, subscribe restart error\n");
4448 4449 4450
        goto PARSE_OVER;
      }
    } else {
4451
      g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4452
    }
4453

4454
    cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress");
4455
    if (keepProgress
4456 4457
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
4458
      if (0 == strcmp("yes", keepProgress->valuestring)) {
4459
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1;
4460
      } else if (0 == strcmp("no", keepProgress->valuestring)) {
4461
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4462
      } else {
4463
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4464 4465 4466
        goto PARSE_OVER;
      }
    } else {
4467
      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4468 4469
    }

4470
    // sqls
4471 4472
    cJSON* specifiedSqls = cJSON_GetObjectItem(specifiedQuery, "sqls");
    if (!specifiedSqls) {
4473
      g_queryInfo.specifiedQueryInfo.sqlCount = 0;
4474
    } else if (specifiedSqls->type != cJSON_Array) {
4475
      errorPrint("%s() LN%d, failed to read json, super sqls not found\n",
4476
          __func__, __LINE__);
4477
      goto PARSE_OVER;
4478
    } else {
4479 4480 4481 4482 4483 4484 4485 4486
      int superSqlSize = cJSON_GetArraySize(specifiedSqls);
      if (superSqlSize * g_queryInfo.specifiedQueryInfo.concurrent
              > MAX_QUERY_SQL_COUNT) {
        errorPrint("%s() LN%d, failed to read json, query sql(%d) * concurrent(%d) overflow, max is %d\n",
           __func__, __LINE__,
           superSqlSize,
           g_queryInfo.specifiedQueryInfo.concurrent,
           MAX_QUERY_SQL_COUNT);
4487 4488
        goto PARSE_OVER;
      }
4489

4490
      g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize;
4491
      for (int j = 0; j < superSqlSize; ++j) {
4492
        cJSON* sql = cJSON_GetArrayItem(specifiedSqls, j);
4493
        if (sql == NULL) continue;
4494

4495 4496
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
4497
          printf("ERROR: failed to read json, sql not found\n");
4498 4499
          goto PARSE_OVER;
        }
4500 4501
        tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j],
                sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
H
hzcheng 已提交
4502

4503 4504
        // default value is -1, which mean infinite loop
        g_queryInfo.specifiedQueryInfo.endAfterConsume[j] = -1;
4505 4506 4507 4508 4509 4510 4511
        cJSON* endAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "endAfterConsume");
        if (endAfterConsume
                && endAfterConsume->type == cJSON_Number) {
            g_queryInfo.specifiedQueryInfo.endAfterConsume[j]
                = endAfterConsume->valueint;
        }
4512 4513
        if (g_queryInfo.specifiedQueryInfo.endAfterConsume[j] < -1)
            g_queryInfo.specifiedQueryInfo.endAfterConsume[j] = -1;
4514

4515
        g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = -1;
4516 4517
        cJSON* resubAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume");
4518 4519 4520
        if ((resubAfterConsume)
                && (resubAfterConsume->type == cJSON_Number)
                && (resubAfterConsume->valueint >= 0)) {
4521 4522 4523 4524
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j]
                = resubAfterConsume->valueint;
        }

4525 4526 4527
        if (g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] < -1)
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = -1;

4528
        cJSON *result = cJSON_GetObjectItem(sql, "result");
4529 4530 4531 4532
        if ((NULL != result) && (result->type == cJSON_String)
                && (result->valuestring != NULL)) {
          tstrncpy(g_queryInfo.specifiedQueryInfo.result[j],
                  result->valuestring, MAX_FILE_NAME_LEN);
4533
        } else if (NULL == result) {
4534 4535
          memset(g_queryInfo.specifiedQueryInfo.result[j],
                  0, MAX_FILE_NAME_LEN);
4536
        } else {
4537
          printf("ERROR: failed to read json, super query result file not found\n");
4538
          goto PARSE_OVER;
4539
        }
4540 4541 4542
      }
    }
  }
4543

4544
  // super_table_query
4545 4546
  cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!superQuery) {
4547
    g_queryInfo.superQueryInfo.threadCnt = 1;
4548 4549
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
4550
    printf("ERROR: failed to read json, sub_table_query not found\n");
4551 4552
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
4553
  } else {
4554
    cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval");
4555
    if (subrate && subrate->type == cJSON_Number) {
4556
      g_queryInfo.superQueryInfo.queryInterval = subrate->valueint;
4557
    } else if (!subrate) {
4558
      g_queryInfo.superQueryInfo.queryInterval = 0;
4559 4560 4561 4562
    }

    cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
    if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
4563 4564
      if (superQueryTimes->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4565
          __func__, __LINE__, superQueryTimes->valueint);
4566 4567
        goto PARSE_OVER;
      }
4568 4569 4570 4571
      g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
    } else if (!superQueryTimes) {
      g_queryInfo.superQueryInfo.queryTimes = g_args.query_times;
    } else {
4572
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4573
          __func__, __LINE__);
4574
      goto PARSE_OVER;
4575
    }
4576

4577
    cJSON* threads = cJSON_GetObjectItem(superQuery, "threads");
4578
    if (threads && threads->type == cJSON_Number) {
4579 4580 4581 4582 4583 4584
      if (threads->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, threads input mistake\n",
          __func__, __LINE__);
        goto PARSE_OVER;

      }
4585
      g_queryInfo.superQueryInfo.threadCnt = threads->valueint;
4586
    } else if (!threads) {
4587
      g_queryInfo.superQueryInfo.threadCnt = 1;
4588
    }
4589

4590
    //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count");
4591
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
4592
    //  g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint;
4593
    //} else if (!subTblCnt) {
4594
    //  g_queryInfo.superQueryInfo.childTblCount = 0;
4595
    //}
4596

4597
    cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname");
4598 4599 4600
    if (stblname && stblname->type == cJSON_String
        && stblname->valuestring != NULL) {
      tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring,
4601
          TSDB_TABLE_NAME_LEN);
4602
    } else {
4603
      errorPrint("%s() LN%d, failed to read json, super table name input error\n",
4604
          __func__, __LINE__);
4605 4606
      goto PARSE_OVER;
    }
4607

4608 4609 4610 4611 4612 4613 4614
    cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode");
    if (superAsyncMode && superAsyncMode->type == cJSON_String
        && superAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE;
4615
      } else {
4616
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4617
            __func__, __LINE__);
4618 4619 4620
        goto PARSE_OVER;
      }
    } else {
4621
      g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
4622
    }
4623

4624 4625 4626 4627 4628 4629 4630 4631 4632
    cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval");
    if (superInterval && superInterval->type == cJSON_Number) {
      if (superInterval->valueint < 0) {
        errorPrint("%s() LN%d, failed to read json, interval input mistake\n",
            __func__, __LINE__);
        goto PARSE_OVER;
      }
      g_queryInfo.superQueryInfo.subscribeInterval = superInterval->valueint;
    } else if (!superInterval) {
4633 4634
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4635
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
4636
    }
4637

4638
    cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart");
4639 4640
    if (subrestart && subrestart->type == cJSON_String
        && subrestart->valuestring != NULL) {
4641
      if (0 == strcmp("yes", subrestart->valuestring)) {
4642
        g_queryInfo.superQueryInfo.subscribeRestart = true;
4643
      } else if (0 == strcmp("no", subrestart->valuestring)) {
4644
        g_queryInfo.superQueryInfo.subscribeRestart = false;
4645
      } else {
4646
        printf("ERROR: failed to read json, subscribe restart error\n");
4647 4648 4649
        goto PARSE_OVER;
      }
    } else {
4650
      g_queryInfo.superQueryInfo.subscribeRestart = true;
4651
    }
4652

4653 4654 4655 4656 4657
    cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
    if (superkeepProgress &&
            superkeepProgress->type == cJSON_String
            && superkeepProgress->valuestring != NULL) {
      if (0 == strcmp("yes", superkeepProgress->valuestring)) {
4658
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
4659
      } else if (0 == strcmp("no", superkeepProgress->valuestring)) {
4660
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4661
      } else {
4662
        printf("ERROR: failed to read json, subscribe super table keepProgress error\n");
4663 4664 4665
        goto PARSE_OVER;
      }
    } else {
4666
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4667
    }
4668

4669 4670
    // default value is -1, which mean do not resub
    g_queryInfo.superQueryInfo.endAfterConsume = -1;
4671 4672 4673 4674 4675 4676 4677
    cJSON* superEndAfterConsume =
            cJSON_GetObjectItem(superQuery, "endAfterConsume");
    if (superEndAfterConsume
            && superEndAfterConsume->type == cJSON_Number) {
        g_queryInfo.superQueryInfo.endAfterConsume =
            superEndAfterConsume->valueint;
    }
4678 4679
    if (g_queryInfo.superQueryInfo.endAfterConsume < -1)
        g_queryInfo.superQueryInfo.endAfterConsume = -1;
4680

4681 4682
    // default value is -1, which mean do not resub
    g_queryInfo.superQueryInfo.resubAfterConsume = -1;
4683
    cJSON* superResubAfterConsume =
4684 4685 4686 4687 4688
            cJSON_GetObjectItem(superQuery, "resubAfterConsume");
    if ((superResubAfterConsume)
            && (superResubAfterConsume->type == cJSON_Number)
            && (superResubAfterConsume->valueint >= 0)) {
        g_queryInfo.superQueryInfo.resubAfterConsume =
4689 4690
            superResubAfterConsume->valueint;
    }
4691 4692
    if (g_queryInfo.superQueryInfo.resubAfterConsume < -1)
        g_queryInfo.superQueryInfo.resubAfterConsume = -1;
4693

4694 4695 4696
    // supert table sqls
    cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls");
    if (!superSqls) {
4697
      g_queryInfo.superQueryInfo.sqlCount = 0;
4698
    } else if (superSqls->type != cJSON_Array) {
4699
      errorPrint("%s() LN%d: failed to read json, super sqls not found\n",
4700
          __func__, __LINE__);
4701
      goto PARSE_OVER;
4702
    } else {
4703
      int superSqlSize = cJSON_GetArraySize(superSqls);
4704
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4705
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
4706
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4707 4708
        goto PARSE_OVER;
      }
4709

4710
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
4711
      for (int j = 0; j < superSqlSize; ++j) {
4712
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
4713
        if (sql == NULL) continue;
4714

4715
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
4716 4717 4718
        if (!sqlStr || sqlStr->type != cJSON_String
            || sqlStr->valuestring == NULL) {
          errorPrint("%s() LN%d, failed to read json, sql not found\n",
4719
              __func__, __LINE__);
4720 4721
          goto PARSE_OVER;
        }
4722
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring,
4723
            MAX_QUERY_SQL_LENGTH);
4724 4725

        cJSON *result = cJSON_GetObjectItem(sql, "result");
4726 4727 4728
        if (result != NULL && result->type == cJSON_String
            && result->valuestring != NULL){
          tstrncpy(g_queryInfo.superQueryInfo.result[j],
4729
              result->valuestring, MAX_FILE_NAME_LEN);
4730
        } else if (NULL == result) {
4731
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4732
        }  else {
4733
          errorPrint("%s() LN%d, failed to read json, sub query result file not found\n",
4734
              __func__, __LINE__);
4735
          goto PARSE_OVER;
4736
        }
4737 4738
      }
    }
H
hzcheng 已提交
4739 4740
  }

4741
  ret = true;
H
hzcheng 已提交
4742

4743
PARSE_OVER:
4744 4745
  return ret;
}
H
hzcheng 已提交
4746

4747
static bool getInfoFromJsonFile(char* file) {
4748
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
4749

4750 4751 4752 4753 4754
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
4755

4756
  bool  ret = false;
4757
  int   maxLen = 6400000;
4758 4759 4760 4761 4762 4763 4764
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
4765
  }
H
Hui Li 已提交
4766

4767 4768 4769
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
4770
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
4771 4772
    goto PARSE_OVER;
  }
H
Hui Li 已提交
4773

4774 4775 4776
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
4777
      g_args.test_mode = INSERT_TEST;
4778
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
4779
      g_args.test_mode = QUERY_TEST;
4780
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
4781
      g_args.test_mode = SUBSCRIBE_TEST;
4782
    } else {
4783
      printf("ERROR: failed to read json, filetype not support\n");
4784 4785 4786
      goto PARSE_OVER;
    }
  } else if (!filetype) {
4787
    g_args.test_mode = INSERT_TEST;
4788
  } else {
4789
    printf("ERROR: failed to read json, filetype not found\n");
4790 4791
    goto PARSE_OVER;
  }
H
hzcheng 已提交
4792

4793
  if (INSERT_TEST == g_args.test_mode) {
4794
    ret = getMetaFromInsertJsonFile(root);
4795
  } else if ((QUERY_TEST == g_args.test_mode)
4796
          || (SUBSCRIBE_TEST == g_args.test_mode)) {
4797 4798
    ret = getMetaFromQueryJsonFile(root);
  } else {
4799
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n",
4800
            __func__, __LINE__);
4801
    goto PARSE_OVER;
4802
  }
4803

4804
PARSE_OVER:
4805 4806
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
4807
  fclose(fp);
4808 4809
  return ret;
}
H
hzcheng 已提交
4810

4811
static int prepareSampleData() {
4812
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4813 4814
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
4815 4816 4817
        if (readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]) != 0) {
          return -1;
        }
S
Shuaiqiang Chang 已提交
4818
      }
4819 4820
    }
  }
4821 4822

  return 0;
4823 4824
}

4825
static void postFreeResource() {
4826
  tmfclose(g_fpOfInsertResult);
4827
  for (int i = 0; i < g_Dbs.dbCount; i++) {
S
Shuduo Sang 已提交
4828
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
4829 4830 4831
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
4832
      }
4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4845
    }
4846 4847
  }
}
S
Shuaiqiang Chang 已提交
4848

4849 4850
static int getRowDataFromSample(
        char* dataBuf, int64_t maxLen, int64_t timestamp,
4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876
      SSuperTable* superTblInfo, int64_t* sampleUsePos)
{
    if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
        /*    int ret = readSampleFromCsvFileToMem(superTblInfo);
              if (0 != ret) {
              tmfree(superTblInfo->sampleDataBuf);
              superTblInfo->sampleDataBuf = NULL;
              return -1;
              }
              */
        *sampleUsePos = 0;
    }

    int    dataLen = 0;

    dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
            "(%" PRId64 ", ", timestamp);
    dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
            "%s",
            superTblInfo->sampleDataBuf
            + superTblInfo->lenOfOneRow * (*sampleUsePos));
    dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");

    (*sampleUsePos)++;

    return dataLen;
4877
}
S
Shuaiqiang Chang 已提交
4878

4879 4880
static int64_t generateStbRowData(
        SSuperTable* stbInfo,
4881 4882
        char* recBuf, int64_t timestamp)
{
4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899
    int64_t   dataLen = 0;
    char  *pstr = recBuf;
    int64_t maxLen = MAX_DATA_SIZE;

    dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
            "(%" PRId64 ",", timestamp);

    for (int i = 0; i < stbInfo->columnCount; i++) {
        if ((0 == strncasecmp(stbInfo->columns[i].dataType,
                        "BINARY", strlen("BINARY")))
                || (0 == strncasecmp(stbInfo->columns[i].dataType,
                        "NCHAR", strlen("NCHAR")))) {
            if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
                errorPrint( "binary or nchar length overflow, max size:%u\n",
                        (uint32_t)TSDB_MAX_BINARY_LEN);
                return -1;
            }
4900

4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959
            char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
            if (NULL == buf) {
                errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
                return -1;
            }
            rand_string(buf, stbInfo->columns[i].dataLen);
            dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf);
            tmfree(buf);
        } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                    "INT", strlen("INT"))) {
            if ((g_args.demo_mode) && (i == 1)) {
                dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
                        "%d,", demo_voltage_int());
            } else {
                dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
                        "%d,", rand_int());
            }
        } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                    "BIGINT", strlen("BIGINT"))) {
            dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
                    "%"PRId64",", rand_bigint());
        }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                    "FLOAT", strlen("FLOAT"))) {
            if (g_args.demo_mode) {
                if (i == 0) {
                    dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
                            "%f,", demo_current_float());
                } else {
                    dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
                            "%f,", demo_phase_float());
                }
            } else {
                dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
                        "%f,", rand_float());
            }
        }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                    "DOUBLE", strlen("DOUBLE"))) {
            dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
                    "%f,", rand_double());
        }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                    "SMALLINT", strlen("SMALLINT"))) {
            dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
                    "%d,", rand_smallint());
        }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                    "TINYINT", strlen("TINYINT"))) {
            dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
                    "%d,", rand_tinyint());
        }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                    "BOOL", strlen("BOOL"))) {
            dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
                    "%d,", rand_bool());
        }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                    "TIMESTAMP", strlen("TIMESTAMP"))) {
            dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
                    "%"PRId64",", rand_bigint());
        }  else {
            errorPrint( "Not support data type: %s\n", stbInfo->columns[i].dataType);
            return -1;
        }
4960
    }
4961

4962 4963
    dataLen -= 1;
    dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")");
4964

4965 4966
    verbosePrint("%s() LN%d, dataLen:%"PRId64"\n", __func__, __LINE__, dataLen);
    verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);
4967

4968
    return strlen(recBuf);
4969
}
S
Shuaiqiang Chang 已提交
4970

4971
static int64_t generateData(char *recBuf, char **data_type,
4972
        int64_t timestamp, int lenOfBinary) {
4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018
    memset(recBuf, 0, MAX_DATA_SIZE);
    char *pstr = recBuf;
    pstr += sprintf(pstr, "(%" PRId64, timestamp);

    int columnCount = g_args.num_of_CPR;

    for (int i = 0; i < columnCount; i++) {
        if (strcasecmp(data_type[i % columnCount], "TINYINT") == 0) {
            pstr += sprintf(pstr, ",%d", rand_tinyint() );
        } else if (strcasecmp(data_type[i % columnCount], "SMALLINT") == 0) {
            pstr += sprintf(pstr, ",%d", rand_smallint());
        } else if (strcasecmp(data_type[i % columnCount], "INT") == 0) {
            pstr += sprintf(pstr, ",%d", rand_int());
        } else if (strcasecmp(data_type[i % columnCount], "BIGINT") == 0) {
            pstr += sprintf(pstr, ",%" PRId64, rand_bigint());
        } else if (strcasecmp(data_type[i % columnCount], "TIMESTAMP") == 0) {
            pstr += sprintf(pstr, ",%" PRId64, rand_bigint());
        } else if (strcasecmp(data_type[i % columnCount], "FLOAT") == 0) {
            pstr += sprintf(pstr, ",%10.4f", rand_float());
        } else if (strcasecmp(data_type[i % columnCount], "DOUBLE") == 0) {
            double t = rand_double();
            pstr += sprintf(pstr, ",%20.8f", t);
        } else if (strcasecmp(data_type[i % columnCount], "BOOL") == 0) {
            bool b = rand_bool() & 1;
            pstr += sprintf(pstr, ",%s", b ? "true" : "false");
        } else if (strcasecmp(data_type[i % columnCount], "BINARY") == 0) {
            char *s = malloc(lenOfBinary + 1);
            if (s == NULL) {
                errorPrint("%s() LN%d, memory allocation %d bytes failed\n",
                        __func__, __LINE__, lenOfBinary + 1);
                exit(-1);
            }
            rand_string(s, lenOfBinary);
            pstr += sprintf(pstr, ",\"%s\"", s);
            free(s);
        } else if (strcasecmp(data_type[i % columnCount], "NCHAR") == 0) {
            char *s = malloc(lenOfBinary + 1);
            if (s == NULL) {
                errorPrint("%s() LN%d, memory allocation %d bytes failed\n",
                        __func__, __LINE__, lenOfBinary + 1);
                exit(-1);
            }
            rand_string(s, lenOfBinary);
            pstr += sprintf(pstr, ",\"%s\"", s);
            free(s);
        }
5019

5020 5021 5022 5023
        if (strlen(recBuf) > MAX_DATA_SIZE) {
            perror("column length too long, abort");
            exit(-1);
        }
5024 5025
    }

5026
    pstr += sprintf(pstr, ")");
5027

5028
    verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);
5029

5030
    return (int32_t)strlen(recBuf);
5031 5032
}

5033
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
5034 5035
  char* sampleDataBuf = NULL;

5036
  sampleDataBuf = calloc(
5037
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
5038
  if (sampleDataBuf == NULL) {
5039
      errorPrint("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n",
5040 5041 5042 5043
              __func__, __LINE__,
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE,
              strerror(errno));
      return -1;
5044
  }
5045

5046 5047 5048 5049
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
5050 5051 5052 5053 5054
      errorPrint("%s() LN%d, read sample from csv file failed.\n",
          __func__, __LINE__);
      tmfree(sampleDataBuf);
      superTblInfo->sampleDataBuf = NULL;
      return -1;
H
Hui Li 已提交
5055
  }
5056

5057 5058 5059
  return 0;
}

5060
static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k)
5061
{
5062 5063
    int32_t affectedRows;
    SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5064

5065
    verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
S
Shuduo Sang 已提交
5066
            __func__, __LINE__, pThreadInfo->buffer);
S
Shuduo Sang 已提交
5067

5068 5069 5070
    uint16_t iface;
    if (superTblInfo)
        iface = superTblInfo->iface;
5071 5072 5073 5074 5075 5076
    else {
        if (g_args.iface == INTERFACE_BUT)
            iface = TAOSC_IFACE;
        else
            iface = g_args.iface;
    }
5077 5078 5079

    debugPrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
            __func__, __LINE__,
5080 5081
            (iface==TAOSC_IFACE)?
            "taosc":(iface==REST_IFACE)?"rest":"stmt");
5082 5083 5084 5085 5086 5087 5088 5089 5090 5091

    switch(iface) {
        case TAOSC_IFACE:
            affectedRows = queryDbExec(
                    pThreadInfo->taos,
                    pThreadInfo->buffer, INSERT_TYPE, false);
            break;

        case REST_IFACE:
            if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port,
5092
                        pThreadInfo->buffer, pThreadInfo)) {
5093 5094 5095 5096 5097 5098 5099 5100
                affectedRows = -1;
                printf("========restful return fail, threadID[%d]\n",
                        pThreadInfo->threadID);
            } else {
                affectedRows = k;
            }
            break;

5101
#if STMT_IFACE_ENABLED == 1
5102 5103 5104 5105 5106 5107 5108 5109 5110
        case STMT_IFACE:
            debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt);
            if (0 != taos_stmt_execute(pThreadInfo->stmt)) {
                errorPrint("%s() LN%d, failied to execute insert statement\n",
                        __func__, __LINE__);
                exit(-1);
            }
            affectedRows = k;
            break;
5111
#endif
5112 5113 5114 5115 5116

        default:
            errorPrint("%s() LN%d: unknown insert mode: %d\n",
                    __func__, __LINE__, superTblInfo->iface);
            affectedRows = 0;
5117
    }
5118

5119
    return affectedRows;
5120 5121
}

5122 5123
static void getTableName(char *pTblName,
        threadInfo* pThreadInfo, uint64_t tableSeq)
5124
{
5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143
    SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
    if (superTblInfo) {
        if (AUTO_CREATE_SUBTBL != superTblInfo->autoCreateTable) {
            if (superTblInfo->childTblLimit > 0) {
                snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
                        superTblInfo->childTblName +
                        (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
            } else {
                verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n",
                        pThreadInfo->threadID, __func__, __LINE__,
                        pThreadInfo->start_table_from,
                        pThreadInfo->ntables, tableSeq);
                snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
                        superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
            }
        } else {
            snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"",
                    superTblInfo->childTblPrefix, tableSeq);
        }
5144
    } else {
5145 5146
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"",
                g_args.tb_prefix, tableSeq);
5147 5148 5149
    }
}

5150
static int32_t generateDataTailWithoutStb(
5151
        uint32_t batch, char* buffer,
5152
        int64_t remainderBufLen, int64_t insertRows,
5153
        uint64_t recordFrom, int64_t startTime,
5154
        /* int64_t *pSamplePos, */int64_t *dataLen) {
5155

5156
  uint64_t len = 0;
5157 5158
  char *pstr = buffer;

5159
  verbosePrint("%s() LN%d batch=%d\n", __func__, __LINE__, batch);
5160

5161
  int32_t k = 0;
5162
  for (k = 0; k < batch;) {
5163
    char data[MAX_DATA_SIZE];
5164 5165
    memset(data, 0, MAX_DATA_SIZE);

5166
    int64_t retLen = 0;
5167

5168 5169
    char **data_type = g_args.datatype;
    int lenOfBinary = g_args.len_of_binary;
5170

5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182
    if (g_args.disorderRatio) {
        retLen = generateData(data, data_type,
                startTime + getTSRandTail(
                    (int64_t) DEFAULT_TIMESTAMP_STEP, k,
                    g_args.disorderRatio,
                    g_args.disorderRange),
                lenOfBinary);
    } else {
        retLen = generateData(data, data_type,
                startTime + (int64_t) (DEFAULT_TIMESTAMP_STEP* k),
                lenOfBinary);
    }
5183

5184
    if (len > remainderBufLen)
5185
        break;
5186

5187 5188 5189 5190 5191
    pstr += sprintf(pstr, "%s", data);
    k++;
    len += retLen;
    remainderBufLen -= retLen;

5192
    verbosePrint("%s() LN%d len=%"PRIu64" k=%d \nbuffer=%s\n",
5193
            __func__, __LINE__, len, k, buffer);
5194

5195
    recordFrom ++;
5196

5197
    if (recordFrom >= insertRows) {
5198 5199 5200
      break;
    }
  }
5201

5202 5203 5204 5205
  *dataLen = len;
  return k;
}

5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222
static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq,
        int disorderRatio, int disorderRange)
{
    int64_t randTail = timeStampStep * seq;
    if (disorderRatio > 0) {
        int rand_num = taosRandom() % 100;
        if(rand_num < disorderRatio) {
            randTail = (randTail +
                    (taosRandom() % disorderRange + 1)) * (-1);
            debugPrint("rand data generated, back %"PRId64"\n", randTail);
        }
    }

    return randTail;
}

static int32_t generateStbDataTail(
5223
        SSuperTable* superTblInfo,
5224
        uint32_t batch, char* buffer,
5225
        int64_t remainderBufLen, int64_t insertRows,
5226
        uint64_t recordFrom, int64_t startTime,
5227 5228 5229 5230 5231
        int64_t *pSamplePos, int64_t *dataLen) {
  uint64_t len = 0;

  char *pstr = buffer;

5232 5233
  bool tsRand;
  if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) {
5234
     tsRand = true;
5235
  } else {
5236
     tsRand = false;
5237
  }
5238 5239
  verbosePrint("%s() LN%d batch=%u buflen=%"PRId64"\n",
          __func__, __LINE__, batch, remainderBufLen);
5240

5241
  int32_t k;
5242 5243 5244 5245
  for (k = 0; k < batch;) {
    char data[MAX_DATA_SIZE];
    memset(data, 0, MAX_DATA_SIZE);

5246
    int64_t lenOfRow = 0;
5247

5248
    if (tsRand) {
5249
        lenOfRow = generateStbRowData(superTblInfo, data,
5250 5251 5252 5253 5254 5255
                startTime + getTSRandTail(
                    superTblInfo->timeStampStep, k,
                    superTblInfo->disorderRatio,
                    superTblInfo->disorderRange)
                );
    } else {
5256
        lenOfRow = getRowDataFromSample(
5257
                  data,
5258
                  (remainderBufLen < MAX_DATA_SIZE)?remainderBufLen:MAX_DATA_SIZE,
5259 5260 5261 5262
                  startTime + superTblInfo->timeStampStep * k,
                  superTblInfo,
                  pSamplePos);
    }
5263

5264
    if ((lenOfRow + 1) > remainderBufLen) {
5265
        break;
5266 5267
    }

5268
    pstr += snprintf(pstr , lenOfRow + 1, "%s", data);
5269
    k++;
5270 5271
    len += lenOfRow;
    remainderBufLen -= lenOfRow;
5272

5273
    verbosePrint("%s() LN%d len=%"PRIu64" k=%u \nbuffer=%s\n",
5274
            __func__, __LINE__, len, k, buffer);
5275

5276
    recordFrom ++;
5277

5278
    if (recordFrom >= insertRows) {
5279
      break;
5280
    }
5281
  }
5282

5283 5284 5285
  *dataLen = len;
  return k;
}
5286

5287 5288 5289

static int generateSQLHeadWithoutStb(char *tableName,
        char *dbName,
5290
        char *buffer, int remainderBufLen)
5291 5292
{
  int len;
5293 5294 5295

  char headBuf[HEAD_BUFF_LEN];

5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321
  len = snprintf(
          headBuf,
          HEAD_BUFF_LEN,
          "%s.%s values",
          dbName,
          tableName);

  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

  return len;
}

static int generateStbSQLHead(
        SSuperTable* superTblInfo,
        char *tableName, int32_t tableSeq,
        char *dbName,
        char *buffer, int remainderBufLen)
{
  int len;

  char headBuf[HEAD_BUFF_LEN];

  if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
5322 5323
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
5324
            tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq);
5325
      } else {
5326 5327 5328
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
5329 5330
      }
      if (NULL == tagsValBuf) {
5331
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n",
5332
            __func__, __LINE__);
5333 5334 5335
        return -1;
      }

5336 5337
      len = snprintf(
          headBuf,
5338 5339
                  HEAD_BUFF_LEN,
                  "%s.%s using %s.%s tags %s values",
5340
                  dbName,
5341
                  tableName,
5342
                  dbName,
5343 5344
                  superTblInfo->sTblName,
                  tagsValBuf);
5345 5346
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
5347 5348
      len = snprintf(
          headBuf,
5349 5350
                  HEAD_BUFF_LEN,
                  "%s.%s values",
5351
                  dbName,
5352
                  tableName);
5353
    } else {
5354 5355
      len = snprintf(
          headBuf,
5356 5357
                  HEAD_BUFF_LEN,
                  "%s.%s values",
5358
                  dbName,
5359
                  tableName);
5360 5361
  }

5362 5363 5364 5365 5366
  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

5367 5368 5369
  return len;
}

5370
static int32_t generateStbInterlaceData(
5371
        SSuperTable *superTblInfo,
5372 5373 5374
        char *tableName, uint32_t batchPerTbl,
        uint64_t i,
        uint32_t batchPerTblTimes,
5375
        uint64_t tableSeq,
5376
        threadInfo *pThreadInfo, char *buffer,
5377
        int64_t insertRows,
5378
        int64_t startTime,
5379
        uint64_t *pRemainderBufLen)
5380
{
5381
  assert(buffer);
5382 5383
  char *pstr = buffer;

5384 5385 5386
  int headLen = generateStbSQLHead(
          superTblInfo,
          tableName, tableSeq, pThreadInfo->db_name,
5387
          pstr, *pRemainderBufLen);
5388 5389 5390 5391 5392

  if (headLen <= 0) {
    return 0;
  }
  // generate data buffer
5393
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n",
5394
            pThreadInfo->threadID, __func__, __LINE__, i, buffer);
5395 5396 5397 5398

  pstr += headLen;
  *pRemainderBufLen -= headLen;

5399
  int64_t dataLen = 0;
5400

5401
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%u batchPerTbl = %u\n",
5402 5403
            pThreadInfo->threadID, __func__, __LINE__,
            i, batchPerTblTimes, batchPerTbl);
5404

5405
  if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
5406
      startTime = taosGetTimestamp(pThreadInfo->time_precision);
5407
  }
5408

5409
  int32_t k = generateStbDataTail(
5410 5411 5412 5413
            superTblInfo,
            batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
            startTime,
            &(pThreadInfo->samplePos), &dataLen);
5414 5415 5416 5417

  if (k == batchPerTbl) {
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
5418
  } else {
5419
    debugPrint("%s() LN%d, generated data tail: %u, not equal batch per table: %u\n",
5420 5421 5422 5423 5424 5425 5426 5427 5428 5429
            __func__, __LINE__, k, batchPerTbl);
    pstr -= headLen;
    pstr[0] = '\0';
    k = 0;
  }

  return k;
}

static int64_t generateInterlaceDataWithoutStb(
5430
        char *tableName, uint32_t batch,
5431 5432 5433
        uint64_t tableSeq,
        char *dbName, char *buffer,
        int64_t insertRows,
5434
        int64_t startTime,
5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452
        uint64_t *pRemainderBufLen)
{
  assert(buffer);
  char *pstr = buffer;

  int headLen = generateSQLHeadWithoutStb(
          tableName, dbName,
            pstr, *pRemainderBufLen);

  if (headLen <= 0) {
    return 0;
  }

  pstr += headLen;
  *pRemainderBufLen -= headLen;

  int64_t dataLen = 0;

5453 5454
  int32_t k = generateDataTailWithoutStb(
            batch, pstr, *pRemainderBufLen, insertRows, 0,
5455
            startTime,
5456
            &dataLen);
5457

5458
  if (k == batch) {
5459 5460 5461
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
  } else {
5462 5463
    debugPrint("%s() LN%d, generated data tail: %d, not equal batch per table: %u\n",
            __func__, __LINE__, k, batch);
5464 5465
    pstr -= headLen;
    pstr[0] = '\0';
5466
    k = 0;
5467 5468 5469 5470 5471
  }

  return k;
}

5472
#if STMT_IFACE_ENABLED == 1
5473
static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind,
5474
        char *dataType, int32_t dataLen, char **ptr, char *value)
5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485
{
    if (0 == strncasecmp(dataType,
                "BINARY", strlen("BINARY"))) {
        if (dataLen > TSDB_MAX_BINARY_LEN) {
            errorPrint( "binary length overflow, max size:%u\n",
                    (uint32_t)TSDB_MAX_BINARY_LEN);
            return -1;
        }
        char *bind_binary = (char *)*ptr;

        bind->buffer_type = TSDB_DATA_TYPE_BINARY;
5486 5487 5488 5489 5490 5491 5492 5493
        if (value) {
            strncpy(bind_binary, value, strlen(value));
            bind->buffer_length = strlen(bind_binary);
        } else {
            rand_string(bind_binary, dataLen);
            bind->buffer_length = dataLen;
        }

5494
        bind->length = &bind->buffer_length;
5495
        bind->buffer = bind_binary;
5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    } else if (0 == strncasecmp(dataType,
                "NCHAR", strlen("NCHAR"))) {
        if (dataLen > TSDB_MAX_BINARY_LEN) {
            errorPrint( "nchar length overflow, max size:%u\n",
                    (uint32_t)TSDB_MAX_BINARY_LEN);
            return -1;
        }
        char *bind_nchar = (char *)*ptr;

        bind->buffer_type = TSDB_DATA_TYPE_NCHAR;
5509 5510 5511 5512 5513 5514
        if (value) {
            strncpy(bind_nchar, value, strlen(value));
        } else {
            rand_string(bind_nchar, dataLen);
        }

5515 5516 5517 5518 5519 5520 5521 5522 5523 5524
        bind->buffer_length = strlen(bind_nchar);
        bind->buffer = bind_nchar;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    } else if (0 == strncasecmp(dataType,
                "INT", strlen("INT"))) {
        int32_t *bind_int = (int32_t *)*ptr;

5525 5526 5527 5528 5529
        if (value) {
            *bind_int = atoi(value);
        } else {
            *bind_int = rand_int();
        }
5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540
        bind->buffer_type = TSDB_DATA_TYPE_INT;
        bind->buffer_length = sizeof(int32_t);
        bind->buffer = bind_int;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    } else if (0 == strncasecmp(dataType,
                "BIGINT", strlen("BIGINT"))) {
        int64_t *bind_bigint = (int64_t *)*ptr;

5541 5542 5543 5544 5545
        if (value) {
            *bind_bigint = atoll(value);
        } else {
            *bind_bigint = rand_bigint();
        }
5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556
        bind->buffer_type = TSDB_DATA_TYPE_BIGINT;
        bind->buffer_length = sizeof(int64_t);
        bind->buffer = bind_bigint;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    }  else if (0 == strncasecmp(dataType,
                "FLOAT", strlen("FLOAT"))) {
        float   *bind_float = (float *) *ptr;

5557 5558 5559 5560 5561
        if (value) {
            *bind_float = (float)atof(value);
        } else {
            *bind_float = rand_float();
        }
5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572
        bind->buffer_type = TSDB_DATA_TYPE_FLOAT;
        bind->buffer_length = sizeof(float);
        bind->buffer = bind_float;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    }  else if (0 == strncasecmp(dataType,
                "DOUBLE", strlen("DOUBLE"))) {
        double  *bind_double = (double *)*ptr;

5573 5574 5575 5576 5577
        if (value) {
            *bind_double = atof(value);
        } else {
            *bind_double = rand_double();
        }
5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588
        bind->buffer_type = TSDB_DATA_TYPE_DOUBLE;
        bind->buffer_length = sizeof(double);
        bind->buffer = bind_double;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    }  else if (0 == strncasecmp(dataType,
                "SMALLINT", strlen("SMALLINT"))) {
        int16_t *bind_smallint = (int16_t *)*ptr;

5589 5590 5591 5592 5593
        if (value) {
            *bind_smallint = (int16_t)atoi(value);
        } else {
            *bind_smallint = rand_smallint();
        }
5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604
        bind->buffer_type = TSDB_DATA_TYPE_SMALLINT;
        bind->buffer_length = sizeof(int16_t);
        bind->buffer = bind_smallint;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    }  else if (0 == strncasecmp(dataType,
                "TINYINT", strlen("TINYINT"))) {
        int8_t  *bind_tinyint = (int8_t *)*ptr;

5605 5606 5607 5608 5609
        if (value) {
            *bind_tinyint = (int8_t)atoi(value);
        } else {
            *bind_tinyint = rand_tinyint();
        }
5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631
        bind->buffer_type = TSDB_DATA_TYPE_TINYINT;
        bind->buffer_length = sizeof(int8_t);
        bind->buffer = bind_tinyint;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;
        *ptr += bind->buffer_length;
    }  else if (0 == strncasecmp(dataType,
                "BOOL", strlen("BOOL"))) {
        int8_t  *bind_bool = (int8_t *)*ptr;

        *bind_bool = rand_bool();
        bind->buffer_type = TSDB_DATA_TYPE_BOOL;
        bind->buffer_length = sizeof(int8_t);
        bind->buffer = bind_bool;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    }  else if (0 == strncasecmp(dataType,
                "TIMESTAMP", strlen("TIMESTAMP"))) {
        int64_t *bind_ts2 = (int64_t *) *ptr;

5632 5633 5634 5635 5636
        if (value) {
            *bind_ts2 = atoll(value);
        } else {
            *bind_ts2 = rand_bigint();
        }
5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653
        bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP;
        bind->buffer_length = sizeof(int64_t);
        bind->buffer = bind_ts2;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        *ptr += bind->buffer_length;
    }  else {
        errorPrint( "No support data type: %s\n",
                dataType);
        return -1;
    }

    return 0;
}

static int32_t prepareStmtWithoutStb(
5654
        TAOS_STMT *stmt,
5655 5656 5657 5658 5659
        char *tableName,
        uint32_t batch,
        int64_t insertRows,
        int64_t recordFrom,
        int64_t startTime)
5660
{
5661 5662 5663 5664 5665 5666 5667 5668
    int ret = taos_stmt_set_tbname(stmt, tableName);
    if (ret != 0) {
        errorPrint("failed to execute taos_stmt_set_tbname(%s). return 0x%x. reason: %s\n",
                tableName, ret, taos_errstr(NULL));
        return ret;
    }

    char **data_type = g_args.datatype;
5669

5670 5671 5672 5673 5674
    char *bindArray = malloc(sizeof(TAOS_BIND) * (g_args.num_of_CPR + 1));
    if (bindArray == NULL) {
        errorPrint("Failed to allocate %d bind params\n",
                (g_args.num_of_CPR + 1));
        return -1;
5675 5676
    }

5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689
    int32_t k = 0;
    for (k = 0; k < batch;) {
        /* columnCount + 1 (ts) */
        char data[MAX_DATA_SIZE];
        memset(data, 0, MAX_DATA_SIZE);

        char *ptr = data;
        TAOS_BIND *bind = (TAOS_BIND *)(bindArray + 0);

        int64_t *bind_ts;

        bind_ts = (int64_t *)ptr;
        bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP;
5690 5691 5692 5693 5694 5695 5696 5697 5698

        if (g_args.disorderRatio) {
            *bind_ts = startTime + getTSRandTail(
                    (int64_t)DEFAULT_TIMESTAMP_STEP, k,
                    g_args.disorderRatio,
                    g_args.disorderRange);
        } else {
            *bind_ts = startTime + (int64_t)(DEFAULT_TIMESTAMP_STEP * k);
        }
5699 5700 5701 5702 5703 5704 5705 5706
        bind->buffer_length = sizeof(int64_t);
        bind->buffer = bind_ts;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        ptr += bind->buffer_length;

        for (int i = 0; i < g_args.num_of_CPR; i ++) {
5707 5708
            bind = (TAOS_BIND *)((char *)bindArray
                    + (sizeof(TAOS_BIND) * (i + 1)));
5709 5710 5711 5712
            if ( -1 == prepareStmtBindArrayByType(
                        bind,
                        data_type[i],
                        g_args.len_of_binary,
5713
                        &ptr, NULL)) {
5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727
                return -1;
            }
        }
        taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray);
        // if msg > 3MB, break
        taos_stmt_add_batch(stmt);

        k++;
        recordFrom ++;
        if (recordFrom >= insertRows) {
            break;
        }
    }

5728
    free(bindArray);
5729 5730 5731
    return k;
}

5732 5733
static int32_t prepareStbStmt(
        SSuperTable *stbInfo,
5734 5735 5736 5737
        TAOS_STMT *stmt,
        char *tableName, uint32_t batch,
        uint64_t insertRows,
        uint64_t recordFrom,
5738 5739
        int64_t startTime,
        int64_t *pSamplePos)
5740 5741 5742 5743 5744
{
    int ret = taos_stmt_set_tbname(stmt, tableName);
    if (ret != 0) {
        errorPrint("failed to execute taos_stmt_set_tbname(%s). return 0x%x. reason: %s\n",
                tableName, ret, taos_errstr(NULL));
5745 5746 5747
        return ret;
    }

5748
    char *bindArray = malloc(sizeof(TAOS_BIND) * (stbInfo->columnCount + 1));
5749
    if (bindArray == NULL) {
5750 5751
        errorPrint("%s() LN%d, Failed to allocate %d bind params\n",
                __func__, __LINE__, (stbInfo->columnCount + 1));
5752 5753 5754
        return -1;
    }

5755
    bool sourceRand;
5756
    if (0 == strncasecmp(stbInfo->dataSource, "rand", strlen("rand"))) {
5757
        sourceRand = true;
5758
    } else {
5759 5760 5761 5762 5763 5764 5765 5766 5767
        sourceRand = false;     // from sample data file
    }

    char *bindBuffer = malloc(g_args.len_of_binary);
    if (bindBuffer == NULL) {
        errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n",
                __func__, __LINE__, g_args.len_of_binary);
        free(bindArray);
        return -1;
5768
    }
5769 5770

    uint32_t k;
5771
    for (k = 0; k < batch;) {
5772
        /* columnCount + 1 (ts) */
5773 5774 5775 5776 5777
        char data[MAX_DATA_SIZE];
        memset(data, 0, MAX_DATA_SIZE);

        char *ptr = data;
        TAOS_BIND *bind = (TAOS_BIND *)(bindArray + 0);
5778

5779 5780 5781 5782
        int64_t *bind_ts;

        bind_ts = (int64_t *)ptr;
        bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP;
5783
        if (stbInfo->disorderRatio) {
5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797
            *bind_ts = startTime + getTSRandTail(
                    stbInfo->timeStampStep, k,
                    stbInfo->disorderRatio,
                    stbInfo->disorderRange);
        } else {
            *bind_ts = startTime + stbInfo->timeStampStep * k;
        }
        bind->buffer_length = sizeof(int64_t);
        bind->buffer = bind_ts;
        bind->length = &bind->buffer_length;
        bind->is_null = NULL;

        ptr += bind->buffer_length;

5798
        int cursor = 0;
5799 5800
        for (int i = 0; i < stbInfo->columnCount; i ++) {
            bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * (i + 1)));
5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837

            if (sourceRand) {
                if ( -1 == prepareStmtBindArrayByType(
                            bind,
                            stbInfo->columns[i].dataType,
                            stbInfo->columns[i].dataLen,
                            &ptr,
                            NULL)) {
                    free(bindArray);
                    free(bindBuffer);
                    return -1;
                }
            } else {
                char *restStr = stbInfo->sampleDataBuf + cursor;
                int lengthOfRest = strlen(restStr);

                int index = 0;
                for (index = 0; index < lengthOfRest; index ++) {
                    if (restStr[index] == ',') {
                        break;
                    }
                }

                memset(bindBuffer, 0, g_args.len_of_binary);
                strncpy(bindBuffer, restStr, index);
                cursor += index + 1; // skip ',' too

                if ( -1 == prepareStmtBindArrayByType(
                            bind,
                            stbInfo->columns[i].dataType,
                            stbInfo->columns[i].dataLen,
                            &ptr,
                            bindBuffer)) {
                    free(bindArray);
                    free(bindBuffer);
                    return -1;
                }
5838 5839
            }
        }
5840
        taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray);
5841
        // if msg > 3MB, break
5842
        taos_stmt_add_batch(stmt);
5843

5844 5845
        k++;
        recordFrom ++;
5846 5847 5848 5849

        if (!sourceRand) {
            (*pSamplePos) ++;
        }
5850

5851 5852 5853 5854
        if (recordFrom >= insertRows) {
            break;
        }
    }
5855

5856
    free(bindBuffer);
5857
    free(bindArray);
5858 5859
    return k;
}
5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896

static int32_t prepareStbStmtInterlace(
        SSuperTable *stbInfo,
        TAOS_STMT *stmt,
        char *tableName, uint32_t batch,
        uint64_t insertRows,
        uint64_t recordFrom,
        int64_t startTime,
        int64_t *pSamplePos)
{
    return prepareStbStmt(
            stbInfo,
            stmt,
            tableName,
            g_args.num_of_RPR,
            insertRows, 0, startTime,
            pSamplePos);
}

static int32_t prepareStbStmtProgressive(
        SSuperTable *stbInfo,
        TAOS_STMT *stmt,
        char *tableName, uint32_t batch,
        uint64_t insertRows,
        uint64_t recordFrom,
        int64_t startTime,
        int64_t *pSamplePos)
{
    return prepareStbStmt(
            stbInfo,
            stmt,
            tableName,
            g_args.num_of_RPR,
            insertRows, recordFrom, startTime,
            pSamplePos);
}

5897
#endif
5898 5899

static int32_t generateStbProgressiveData(
5900
        SSuperTable *superTblInfo,
5901
        char *tableName,
5902
        int64_t tableSeq,
5903
        char *dbName, char *buffer,
5904
        int64_t insertRows,
5905
        uint64_t recordFrom, int64_t startTime, int64_t *pSamplePos,
5906
        int64_t *pRemainderBufLen)
5907 5908
{
  assert(buffer != NULL);
5909
  char *pstr = buffer;
5910

5911
  memset(buffer, 0, *pRemainderBufLen);
5912

5913
  int64_t headLen = generateStbSQLHead(
5914
              superTblInfo,
5915
          tableName, tableSeq, dbName,
5916
          buffer, *pRemainderBufLen);
5917 5918 5919 5920

  if (headLen <= 0) {
    return 0;
  }
5921
  pstr += headLen;
5922
  *pRemainderBufLen -= headLen;
5923

5924
  int64_t dataLen;
5925

5926
  return generateStbDataTail(superTblInfo,
5927
          g_args.num_of_RPR, pstr, *pRemainderBufLen,
5928
          insertRows, recordFrom,
5929 5930
          startTime,
          pSamplePos, &dataLen);
5931 5932
}

5933
static int32_t generateProgressiveDataWithoutStb(
5934
        char *tableName,
5935
        /* int64_t tableSeq, */
5936 5937
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
5938
        uint64_t recordFrom, int64_t startTime, /*int64_t *pSamplePos, */
5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958
        int64_t *pRemainderBufLen)
{
  assert(buffer != NULL);
  char *pstr = buffer;

  memset(buffer, 0, *pRemainderBufLen);

  int64_t headLen = generateSQLHeadWithoutStb(
          tableName, pThreadInfo->db_name,
          buffer, *pRemainderBufLen);

  if (headLen <= 0) {
    return 0;
  }
  pstr += headLen;
  *pRemainderBufLen -= headLen;

  int64_t dataLen;

  return generateDataTailWithoutStb(
5959
          g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, recordFrom,
5960 5961
          startTime,
          /*pSamplePos, */&dataLen);
5962
}
5963

5964 5965
static void printStatPerThread(threadInfo *pThreadInfo)
{
5966
    fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n",
5967 5968 5969 5970 5971 5972
            pThreadInfo->threadID,
            pThreadInfo->totalInsertRows,
            pThreadInfo->totalAffectedRows,
            (pThreadInfo->totalDelay/1000.0)?
            (double)(pThreadInfo->totalAffectedRows/(pThreadInfo->totalDelay/1000.0)):
            FLT_MAX);
5973 5974
}

5975
// sync write interlace data
5976
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
5977
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
5978 5979
         pThreadInfo->threadID, __func__, __LINE__);

5980
  int64_t insertRows;
5981
  uint32_t interlaceRows;
5982 5983 5984
  uint64_t maxSqlLen;
  int64_t nTimeStampStep;
  uint64_t insert_interval;
5985 5986

  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5987

5988 5989 5990 5991 5992 5993 5994 5995 5996
  if (superTblInfo) {
    insertRows = superTblInfo->insertRows;

    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
5997 5998 5999
    maxSqlLen = superTblInfo->maxSqlLen;
    nTimeStampStep = superTblInfo->timeStampStep;
    insert_interval = superTblInfo->insertInterval;
6000 6001 6002
  } else {
    insertRows = g_args.num_of_DPT;
    interlaceRows = g_args.interlace_rows;
6003 6004 6005
    maxSqlLen = g_args.max_sql_len;
    nTimeStampStep = DEFAULT_TIMESTAMP_STEP;
    insert_interval = g_args.insert_interval;
6006
  }
6007

6008 6009 6010 6011 6012
  debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n",
          pThreadInfo->threadID, __func__, __LINE__,
          pThreadInfo->start_table_from,
          pThreadInfo->ntables, insertRows);

6013
  if (interlaceRows > insertRows)
H
Haojun Liao 已提交
6014
    interlaceRows = insertRows;
6015

6016 6017
  if (interlaceRows > g_args.num_of_RPR)
    interlaceRows = g_args.num_of_RPR;
6018

6019 6020
  uint32_t batchPerTbl = interlaceRows;
  uint32_t batchPerTblTimes;
6021

6022 6023 6024
  if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) {
    batchPerTblTimes =
        g_args.num_of_RPR / interlaceRows;
6025
  } else {
6026
    batchPerTblTimes = 1;
6027 6028
  }

S
Shuduo Sang 已提交
6029 6030
  pThreadInfo->buffer = calloc(maxSqlLen, 1);
  if (NULL == pThreadInfo->buffer) {
6031
    errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n",
6032
              __func__, __LINE__, maxSqlLen, strerror(errno));
6033 6034 6035 6036 6037 6038
    return NULL;
  }

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

6039 6040
  uint64_t st = 0;
  uint64_t et = UINT64_MAX;
6041

6042 6043 6044
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
6045

6046
  uint64_t tableSeq = pThreadInfo->start_table_from;
6047 6048
  int64_t startTime = pThreadInfo->start_time;

6049
  uint64_t generatedRecPerTbl = 0;
6050
  bool flagSleep = true;
6051
  uint64_t sleepTimeTotal = 0;
6052

6053
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
6054
    if ((flagSleep) && (insert_interval)) {
6055 6056
        st = taosGetTimestampMs();
        flagSleep = false;
6057 6058
    }
    // generate data
S
Shuduo Sang 已提交
6059
    memset(pThreadInfo->buffer, 0, maxSqlLen);
6060
    uint64_t remainderBufLen = maxSqlLen;
6061

S
Shuduo Sang 已提交
6062
    char *pstr = pThreadInfo->buffer;
6063

6064 6065
    int len = snprintf(pstr,
            strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO);
6066 6067 6068
    pstr += len;
    remainderBufLen -= len;

6069
    uint32_t recOfBatch = 0;
6070

6071
    for (uint64_t i = 0; i < batchPerTblTimes; i ++) {
6072 6073
        char tableName[TSDB_TABLE_NAME_LEN];

6074
      getTableName(tableName, pThreadInfo, tableSeq);
6075 6076
      if (0 == strlen(tableName)) {
        errorPrint("[%d] %s() LN%d, getTableName return null\n",
6077
            pThreadInfo->threadID, __func__, __LINE__);
S
Shuduo Sang 已提交
6078
        free(pThreadInfo->buffer);
6079 6080
        return NULL;
      }
6081

6082
      uint64_t oldRemainderLen = remainderBufLen;
6083

6084
      int32_t generated;
6085
      if (superTblInfo) {
6086
          if (superTblInfo->iface == STMT_IFACE) {
6087
#if STMT_IFACE_ENABLED == 1
6088
              generated = prepareStbStmtInterlace(
6089
                      superTblInfo,
6090 6091 6092 6093 6094
                      pThreadInfo->stmt,
                      tableName,
                      batchPerTbl,
                      insertRows, i,
                      startTime,
6095
                      &(pThreadInfo->samplePos));
6096 6097 6098
#else
              generated = -1;
#endif
6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109
          } else {
              generated = generateStbInterlaceData(
                      superTblInfo,
                      tableName, batchPerTbl, i,
                      batchPerTblTimes,
                      tableSeq,
                      pThreadInfo, pstr,
                      insertRows,
                      startTime,
                      &remainderBufLen);
          }
6110
      } else {
6111 6112 6113 6114 6115
          if (g_args.iface == STMT_IFACE) {
              debugPrint("[%d] %s() LN%d, tableName:%s, batch:%d startTime:%"PRId64"\n",
                      pThreadInfo->threadID,
                      __func__, __LINE__,
                      tableName, batchPerTbl, startTime);
6116
#if STMT_IFACE_ENABLED == 1
6117 6118 6119 6120 6121
              generated = prepareStmtWithoutStb(
                      pThreadInfo->stmt, tableName,
                      batchPerTbl,
                      insertRows, i,
                      startTime);
6122 6123 6124
#else
              generated = -1;
#endif
6125 6126 6127 6128 6129 6130 6131 6132 6133
          } else {
              generated = generateInterlaceDataWithoutStb(
                      tableName, batchPerTbl,
                      tableSeq,
                      pThreadInfo->db_name, pstr,
                      insertRows,
                      startTime,
                      &remainderBufLen);
          }
6134
      }
6135

6136
      debugPrint("[%d] %s() LN%d, generated records is %d\n",
6137
                  pThreadInfo->threadID, __func__, __LINE__, generated);
6138
      if (generated < 0) {
6139
        errorPrint("[%d] %s() LN%d, generated records is %d\n",
6140
                  pThreadInfo->threadID, __func__, __LINE__, generated);
6141
        goto free_of_interlace;
6142 6143
      } else if (generated == 0) {
        break;
6144 6145
      }

6146
      tableSeq ++;
6147
      recOfBatch += batchPerTbl;
6148

6149
      pstr += (oldRemainderLen - remainderBufLen);
6150
      pThreadInfo->totalInsertRows += batchPerTbl;
6151

6152
      verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n",
6153 6154
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
6155

6156
      if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
6157 6158 6159
            // turn to first table
            tableSeq = pThreadInfo->start_table_from;
            generatedRecPerTbl += batchPerTbl;
6160

6161 6162
            startTime = pThreadInfo->start_time
              + generatedRecPerTbl * nTimeStampStep;
6163

6164 6165 6166
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;
6167

6168
            int64_t remainRows = insertRows - generatedRecPerTbl;
6169 6170
            if ((remainRows > 0) && (batchPerTbl > remainRows))
              batchPerTbl = remainRows;
6171

6172 6173 6174
            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
      }
6175

6176
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n",
6177 6178
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);
6179 6180

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
6181 6182 6183
        break;
    }

6184
    verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRIu64"\n",
6185 6186
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
6187
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
S
Shuduo Sang 已提交
6188
           pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer);
6189

6190
    startTs = taosGetTimestampMs();
6191

6192
    if (recOfBatch == 0) {
6193 6194 6195
        errorPrint("[%d] %s() LN%d Failed to insert records of batch %d\n",
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl);
6196 6197 6198 6199
        if (batchPerTbl > 0) {
          errorPrint("\tIf the batch is %d, the length of the SQL to insert a row must be less then %"PRId64"\n",
              batchPerTbl, maxSqlLen / batchPerTbl);
        }
6200 6201 6202
        errorPrint("\tPlease check if the buffer length(%"PRId64") or batch(%d) is set with proper value!\n",
                maxSqlLen, batchPerTbl);
        goto free_of_interlace;
6203
    }
S
Shuduo Sang 已提交
6204
    int64_t affectedRows = execInsert(pThreadInfo, recOfBatch);
6205

6206
    endTs = taosGetTimestampMs();
6207 6208
    uint64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n",
6209
            __func__, __LINE__, delay);
6210 6211 6212
    verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
6213 6214 6215 6216 6217 6218

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

6219
    if (recOfBatch != affectedRows) {
6220
        errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %"PRId64"\n%s\n",
6221
                pThreadInfo->threadID, __func__, __LINE__,
S
Shuduo Sang 已提交
6222
                recOfBatch, affectedRows, pThreadInfo->buffer);
6223
        goto free_of_interlace;
6224
    }
6225

6226
    pThreadInfo->totalAffectedRows += affectedRows;
6227

6228 6229
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
6230
      printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n",
6231 6232 6233
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
6234 6235
      lastPrintTime = currentPrintTime;
    }
6236

6237
    if ((insert_interval) && flagSleep) {
6238
      et = taosGetTimestampMs();
6239

6240
      if (insert_interval > (et - st) ) {
S
Shuduo Sang 已提交
6241 6242
        uint64_t sleepTime = insert_interval - (et -st);
        performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n",
6243
                    __func__, __LINE__, sleepTime);
6244 6245 6246
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
6247
    }
6248 6249
  }

6250
free_of_interlace:
S
Shuduo Sang 已提交
6251
  tmfree(pThreadInfo->buffer);
6252
  printStatPerThread(pThreadInfo);
6253 6254 6255
  return NULL;
}

6256
// sync insertion progressive data
6257
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
6258
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
6259

6260
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
6261
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
6262 6263 6264 6265 6266 6267
  int64_t timeStampStep =
      superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
  int64_t insertRows =
        (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
  verbosePrint("%s() LN%d insertRows=%"PRId64"\n",
            __func__, __LINE__, insertRows);
6268

S
Shuduo Sang 已提交
6269 6270
  pThreadInfo->buffer = calloc(maxSqlLen, 1);
  if (NULL == pThreadInfo->buffer) {
6271
    errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n",
6272 6273
              maxSqlLen,
              strerror(errno));
6274 6275
    return NULL;
  }
6276

6277 6278 6279
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
6280

6281 6282
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
6283

6284
  pThreadInfo->samplePos = 0;
6285

6286 6287 6288
  for (uint64_t tableSeq = pThreadInfo->start_table_from;
          tableSeq <= pThreadInfo->end_table_to;
          tableSeq ++) {
6289
    int64_t start_time = pThreadInfo->start_time;
6290

6291
    for (uint64_t i = 0; i < insertRows;) {
6292 6293
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
6294
      verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n",
6295 6296
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);
6297 6298 6299 6300 6301 6302
      if (0 == strlen(tableName)) {
        errorPrint("[%d] %s() LN%d, getTableName return null\n",
            pThreadInfo->threadID, __func__, __LINE__);
        free(pThreadInfo->buffer);
        return NULL;
      }
6303

6304
      int64_t remainderBufLen = maxSqlLen;
S
Shuduo Sang 已提交
6305
      char *pstr = pThreadInfo->buffer;
6306

6307 6308
      int len = snprintf(pstr,
              strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO);
6309 6310 6311 6312

      pstr += len;
      remainderBufLen -= len;

6313
      int32_t generated;
6314
      if (superTblInfo) {
6315
          if (superTblInfo->iface == STMT_IFACE) {
6316
#if STMT_IFACE_ENABLED == 1
6317
              generated = prepareStbStmtProgressive(
6318
                      superTblInfo,
6319
                      pThreadInfo->stmt,
6320 6321
                      tableName,
                      g_args.num_of_RPR,
6322 6323
                      insertRows, i, start_time,
                      &(pThreadInfo->samplePos));
6324 6325 6326
#else
              generated = -1;
#endif
6327 6328 6329 6330 6331 6332 6333 6334
          } else {
              generated = generateStbProgressiveData(
                      superTblInfo,
                      tableName, tableSeq, pThreadInfo->db_name, pstr,
                      insertRows, i, start_time,
                      &(pThreadInfo->samplePos),
                      &remainderBufLen);
          }
6335
      } else {
6336
          if (g_args.iface == STMT_IFACE) {
6337
#if STMT_IFACE_ENABLED == 1
6338 6339 6340 6341 6342 6343
              generated = prepareStmtWithoutStb(
                      pThreadInfo->stmt,
                      tableName,
                      g_args.num_of_RPR,
                      insertRows, i,
                      start_time);
6344 6345 6346
#else
              generated = -1;
#endif
6347 6348 6349 6350 6351 6352 6353 6354 6355
          } else {
              generated = generateProgressiveDataWithoutStb(
                      tableName,
                      /*  tableSeq, */
                      pThreadInfo, pstr, insertRows,
                      i, start_time,
                      /* &(pThreadInfo->samplePos), */
                      &remainderBufLen);
          }
6356
      }
6357 6358 6359
      if (generated > 0)
        i += generated;
      else
6360
        goto free_of_progressive;
6361

6362
      start_time +=  generated * timeStampStep;
6363
      pThreadInfo->totalInsertRows += generated;
6364

6365
      startTs = taosGetTimestampMs();
6366

6367
      int32_t affectedRows = execInsert(pThreadInfo, generated);
6368

6369
      endTs = taosGetTimestampMs();
6370
      uint64_t delay = endTs - startTs;
6371
      performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n",
6372
              __func__, __LINE__, delay);
6373
      verbosePrint("[%d] %s() LN%d affectedRows=%d\n",
6374 6375
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
6376

6377 6378 6379 6380
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
6381

6382
      if (affectedRows < 0) {
6383
        errorPrint("%s() LN%d, affected rows: %d\n",
6384 6385 6386
                __func__, __LINE__, affectedRows);
        goto free_of_progressive;
      }
6387 6388 6389

      pThreadInfo->totalAffectedRows += affectedRows;

6390 6391
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
6392
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
6393 6394 6395
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
6396 6397 6398
        lastPrintTime = currentPrintTime;
      }

6399
      if (i >= insertRows)
6400 6401 6402
        break;
    }   // num_of_DPT

6403 6404
    if ((g_args.verbose_print) &&
      (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) &&
6405 6406
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
6407
          verbosePrint("%s() LN%d samplePos=%"PRId64"\n",
6408
                  __func__, __LINE__, pThreadInfo->samplePos);
6409
    }
6410
  } // tableSeq
6411

6412
free_of_progressive:
S
Shuduo Sang 已提交
6413
  tmfree(pThreadInfo->buffer);
6414
  printStatPerThread(pThreadInfo);
6415
  return NULL;
H
Hui Li 已提交
6416 6417
}

6418 6419
static void* syncWrite(void *sarg) {

6420 6421
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
6422

6423
  uint32_t interlaceRows;
6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434

  if (superTblInfo) {
    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    interlaceRows = g_args.interlace_rows;
  }
6435

6436
  if (interlaceRows > 0) {
6437
    // interlace mode
6438
    return syncWriteInterlace(pThreadInfo);
6439 6440
  } else {
    // progressive mode
6441
    return syncWriteProgressive(pThreadInfo);
6442 6443 6444
  }
}

6445
static void callBack(void *param, TAOS_RES *res, int code) {
6446 6447
  threadInfo* pThreadInfo = (threadInfo*)param;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
6448

6449 6450
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
6451
  if (insert_interval) {
6452 6453 6454
    pThreadInfo->et = taosGetTimestampMs();
    if ((pThreadInfo->et - pThreadInfo->st) < insert_interval) {
      taosMsleep(insert_interval - (pThreadInfo->et - pThreadInfo->st)); // ms
6455
    }
H
Hui Li 已提交
6456
  }
6457

6458
  char *buffer = calloc(1, pThreadInfo->superTblInfo->maxSqlLen);
6459
  char data[MAX_DATA_SIZE];
6460
  char *pstr = buffer;
6461 6462
  pstr += sprintf(pstr, "insert into %s.%s%"PRId64" values",
          pThreadInfo->db_name, pThreadInfo->tb_prefix,
6463
          pThreadInfo->start_table_from);
6464 6465 6466 6467 6468 6469 6470
//  if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
  if (pThreadInfo->counter >= g_args.num_of_RPR) {
    pThreadInfo->start_table_from++;
    pThreadInfo->counter = 0;
  }
  if (pThreadInfo->start_table_from > pThreadInfo->end_table_to) {
    tsem_post(&pThreadInfo->lock_sem);
6471 6472 6473 6474
    free(buffer);
    taos_free_result(res);
    return;
  }
6475

6476
  for (int i = 0; i < g_args.num_of_RPR; i++) {
6477
    int rand_num = taosRandom() % 100;
6478
    if (0 != pThreadInfo->superTblInfo->disorderRatio
6479
            && rand_num < pThreadInfo->superTblInfo->disorderRatio) {
6480 6481 6482
      int64_t d = pThreadInfo->lastTs
          - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1);
      generateStbRowData(pThreadInfo->superTblInfo, data, d);
6483
    } else {
6484 6485
      generateStbRowData(pThreadInfo->superTblInfo,
              data, pThreadInfo->lastTs += 1000);
H
Hui Li 已提交
6486
    }
6487
    pstr += sprintf(pstr, "%s", data);
6488
    pThreadInfo->counter++;
H
Hui Li 已提交
6489

6490
    if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
6491
      break;
H
Hui Li 已提交
6492 6493
    }
  }
6494

6495
  if (insert_interval) {
6496
    pThreadInfo->st = taosGetTimestampMs();
6497
  }
6498
  taos_query_a(pThreadInfo->taos, buffer, callBack, pThreadInfo);
6499
  free(buffer);
H
Hui Li 已提交
6500

6501
  taos_free_result(res);
H
Hui Li 已提交
6502 6503
}

6504
static void *asyncWrite(void *sarg) {
6505 6506
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
6507

6508 6509 6510
  pThreadInfo->st = 0;
  pThreadInfo->et = 0;
  pThreadInfo->lastTs = pThreadInfo->start_time;
6511

6512
  int insert_interval =
6513
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
6514
  if (insert_interval) {
6515
    pThreadInfo->st = taosGetTimestampMs();
H
Hui Li 已提交
6516
  }
6517
  taos_query_a(pThreadInfo->taos, "show databases", callBack, pThreadInfo);
6518

6519
  tsem_wait(&(pThreadInfo->lock_sem));
H
Hui Li 已提交
6520 6521 6522 6523

  return NULL;
}

6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549
static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in *serv_addr)
{
  uint16_t rest_port = port + TSDB_PORT_HTTP;
  struct hostent *server = gethostbyname(host);
  if ((server == NULL) || (server->h_addr == NULL)) {
    errorPrint("%s", "ERROR, no such host");
    return -1;
  }

  debugPrint("h_name: %s\nh_addr=%p\nh_addretype: %s\nh_length: %d\n",
            server->h_name,
            server->h_addr,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

  memset(serv_addr, 0, sizeof(struct sockaddr_in));
  serv_addr->sin_family = AF_INET;
  serv_addr->sin_port = htons(rest_port);
#ifdef WINDOWS
  serv_addr->sin_addr.s_addr = inet_addr(host);
#else
  memcpy(&(serv_addr->sin_addr.s_addr), server->h_addr, server->h_length);
#endif
  return 0;
}

6550
static void startMultiThreadInsertData(int threads, char* db_name,
6551
        char* precision, SSuperTable* superTblInfo) {
6552

6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566
    int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
    if (0 != precision[0]) {
        if (0 == strncasecmp(precision, "ms", 2)) {
            timePrec = TSDB_TIME_PRECISION_MILLI;
        } else if (0 == strncasecmp(precision, "us", 2)) {
            timePrec = TSDB_TIME_PRECISION_MICRO;
#if NANO_SECOND_ENABLED == 1
        } else if (0 == strncasecmp(precision, "ns", 2)) {
            timePrec = TSDB_TIME_PRECISION_NANO;
#endif
        } else {
            errorPrint("Not support precision: %s\n", precision);
            exit(-1);
        }
6567 6568
    }

6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581
    int64_t start_time;
    if (superTblInfo) {
        if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
            start_time = taosGetTimestamp(timePrec);
        } else {
            if (TSDB_CODE_SUCCESS != taosParseTime(
                        superTblInfo->startTimestamp,
                        &start_time,
                        strlen(superTblInfo->startTimestamp),
                        timePrec, 0)) {
                ERROR_EXIT("failed to parse time!\n");
            }
        }
6582
    } else {
6583
        start_time = 1500000000000;
6584
    }
6585

6586
    int64_t start = taosGetTimestampMs();
6587

6588 6589 6590 6591 6592 6593 6594 6595
    // read sample data from file first
    if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
                    "sample", strlen("sample")))) {
        if (0 != prepareSampleDataForSTable(superTblInfo)) {
            errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
                    __func__, __LINE__);
            exit(-1);
        }
6596 6597
    }

6598 6599 6600 6601 6602
    TAOS* taos0 = taos_connect(
            g_Dbs.host, g_Dbs.user,
            g_Dbs.password, db_name, g_Dbs.port);
    if (NULL == taos0) {
        errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
6603
                __func__, __LINE__, taos_errstr(NULL));
6604 6605
        exit(-1);
    }
6606

6607 6608
    int64_t ntables = 0;
    uint64_t tableFrom;
6609

6610 6611 6612
    if (superTblInfo) {
        int64_t limit;
        uint64_t offset;
6613

6614 6615 6616 6617
        if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) &&
                ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) {
            printf("WARNING: offset and limit will not be used since the child tables not exists!\n");
        }
6618

6619 6620 6621 6622 6623 6624 6625
        if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) {
            if ((superTblInfo->childTblLimit < 0)
                    || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit)
                        > (superTblInfo->childTblCount))) {
                superTblInfo->childTblLimit =
                    superTblInfo->childTblCount - superTblInfo->childTblOffset;
            }
6626

6627 6628 6629 6630 6631 6632
            offset = superTblInfo->childTblOffset;
            limit = superTblInfo->childTblLimit;
        } else {
            limit = superTblInfo->childTblCount;
            offset = 0;
        }
6633

6634 6635
        ntables = limit;
        tableFrom = offset;
6636

6637 6638 6639 6640 6641 6642
        if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
                && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
                    > superTblInfo->childTblCount)) {
            printf("WARNING: specified offset + limit > child table count!\n");
            prompt();
        }
6643

6644 6645 6646 6647 6648
        if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
                && (0 == superTblInfo->childTblLimit)) {
            printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
            prompt();
        }
6649

6650 6651 6652 6653 6654 6655 6656
        superTblInfo->childTblName = (char*)calloc(1,
                limit * TSDB_TABLE_NAME_LEN);
        if (superTblInfo->childTblName == NULL) {
            errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
            taos_close(taos0);
            exit(-1);
        }
6657

6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668
        int64_t childTblCount;
        getChildNameOfSuperTableWithLimitAndOffset(
                taos0,
                db_name, superTblInfo->sTblName,
                &superTblInfo->childTblName, &childTblCount,
                limit,
                offset);
    } else {
        ntables = g_args.num_of_tables;
        tableFrom = 0;
    }
6669

6670
    taos_close(taos0);
6671

6672 6673 6674 6675 6676
    int64_t a = ntables / threads;
    if (a < 1) {
        threads = ntables;
        a = 1;
    }
6677

6678 6679 6680 6681
    int64_t b = 0;
    if (threads != 0) {
        b = ntables % threads;
    }
6682

6683 6684 6685 6686 6687 6688 6689
    if ((superTblInfo)
            && (superTblInfo->iface == REST_IFACE)) {
        if (convertHostToServAddr(
                    g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) {
            exit(-1);
        }
    }
6690

6691 6692
    pthread_t *pids = calloc(1, threads * sizeof(pthread_t));
    assert(pids != NULL);
6693

6694 6695
    threadInfo *infos = calloc(1, threads * sizeof(threadInfo));
    assert(infos != NULL);
6696

6697 6698
    memset(pids, 0, threads * sizeof(pthread_t));
    memset(infos, 0, threads * sizeof(threadInfo));
6699

6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721 6722 6723
    for (int i = 0; i < threads; i++) {
        threadInfo *pThreadInfo = infos + i;
        pThreadInfo->threadID = i;
        tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN);
        pThreadInfo->time_precision = timePrec;
        pThreadInfo->superTblInfo = superTblInfo;

        pThreadInfo->start_time = start_time;
        pThreadInfo->minDelay = UINT64_MAX;

        if ((NULL == superTblInfo) ||
                (superTblInfo->iface != REST_IFACE)) {
            //t_info->taos = taos;
            pThreadInfo->taos = taos_connect(
                    g_Dbs.host, g_Dbs.user,
                    g_Dbs.password, db_name, g_Dbs.port);
            if (NULL == pThreadInfo->taos) {
                errorPrint(
                        "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n",
                        __func__, __LINE__,
                        taos_errstr(NULL));
                free(infos);
                exit(-1);
            }
6724

6725 6726 6727
#if STMT_IFACE_ENABLED == 1
            if ((g_args.iface == STMT_IFACE)
                    || ((superTblInfo) && (superTblInfo->iface == STMT_IFACE))) {
6728

6729 6730 6731 6732 6733 6734
                int columnCount;
                if (superTblInfo) {
                    columnCount = superTblInfo->columnCount;
                } else {
                    columnCount = g_args.num_of_CPR;
                }
6735

6736 6737 6738 6739 6740 6741 6742 6743 6744 6745
                pThreadInfo->stmt = taos_stmt_init(pThreadInfo->taos);
                if (NULL == pThreadInfo->stmt) {
                    errorPrint(
                            "%s() LN%d, failed init stmt, reason: %s\n",
                            __func__, __LINE__,
                            taos_errstr(NULL));
                    free(pids);
                    free(infos);
                    exit(-1);
                }
6746

6747 6748
                char buffer[3000];
                char *pstr = buffer;
6749

6750 6751 6752
                if ((superTblInfo)
                        && (AUTO_CREATE_SUBTBL
                            == superTblInfo->autoCreateTable)) {
6753 6754 6755 6756 6757 6758 6759 6760 6761
                    pstr += sprintf(pstr, "INSERT INTO ? USING %s TAGS(?",
                            superTblInfo->sTblName);
                    for (int tag = 0; tag < (superTblInfo->tagCount - 1); tag ++ ) {
                        pstr += sprintf(pstr, ",?");
                    }
                    pstr += sprintf(pstr, ") VALUES(?");
                } else {
                    pstr += sprintf(pstr, "INSERT INTO ? VALUES(?");
                }
6762

6763 6764 6765 6766
                for (int col = 0; col < columnCount; col ++) {
                    pstr += sprintf(pstr, ",?");
                }
                pstr += sprintf(pstr, ")");
6767

6768 6769 6770 6771 6772 6773 6774 6775 6776
                debugPrint("%s() LN%d, buffer: %s", __func__, __LINE__, buffer);
                int ret = taos_stmt_prepare(pThreadInfo->stmt, buffer, 0);
                if (ret != 0){
                    errorPrint("failed to execute taos_stmt_prepare. return 0x%x. reason: %s\n",
                            ret, taos_errstr(NULL));
                    free(pids);
                    free(infos);
                    exit(-1);
                }
6777
            }
6778
#endif
6779
        } else {
6780
            pThreadInfo->taos = NULL;
6781
        }
6782

6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 6799 6800
        /*    if ((NULL == superTblInfo)
              || (0 == superTblInfo->multiThreadWriteOneTbl)) {
              */
        pThreadInfo->start_table_from = tableFrom;
        pThreadInfo->ntables = i<b?a+1:a;
        pThreadInfo->end_table_to = i < b ? tableFrom + a : tableFrom + a - 1;
        tableFrom = pThreadInfo->end_table_to + 1;
        /*    } else {
              pThreadInfo->start_table_from = 0;
              pThreadInfo->ntables = superTblInfo->childTblCount;
              pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint();
              }
              */
        tsem_init(&(pThreadInfo->lock_sem), 0, 0);
        if (ASYNC_MODE == g_Dbs.asyncMode) {
            pthread_create(pids + i, NULL, asyncWrite, pThreadInfo);
        } else {
            pthread_create(pids + i, NULL, syncWrite, pThreadInfo);
6801
        }
6802 6803
    }

6804 6805
    for (int i = 0; i < threads; i++) {
        pthread_join(pids[i], NULL);
6806
    }
H
Hui Li 已提交
6807

6808 6809 6810 6811 6812
    uint64_t totalDelay = 0;
    uint64_t maxDelay = 0;
    uint64_t minDelay = UINT64_MAX;
    uint64_t cntDelay = 1;
    double  avgDelay = 0;
6813

6814 6815
    for (int i = 0; i < threads; i++) {
        threadInfo *pThreadInfo = infos + i;
6816

6817
        tsem_destroy(&(pThreadInfo->lock_sem));
6818

6819
#if STMT_IFACE_ENABLED == 1
6820 6821 6822
        if (pThreadInfo->stmt) {
            taos_stmt_close(pThreadInfo->stmt);
        }
6823
#endif
6824 6825
        tsem_destroy(&(pThreadInfo->lock_sem));
        taos_close(pThreadInfo->taos);
6826

6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837
        debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n",
                __func__, __LINE__,
                pThreadInfo->threadID, pThreadInfo->totalInsertRows,
                pThreadInfo->totalAffectedRows);
        if (superTblInfo) {
            superTblInfo->totalAffectedRows += pThreadInfo->totalAffectedRows;
            superTblInfo->totalInsertRows += pThreadInfo->totalInsertRows;
        } else {
            g_args.totalAffectedRows += pThreadInfo->totalAffectedRows;
            g_args.totalInsertRows += pThreadInfo->totalInsertRows;
        }
H
Hui Li 已提交
6838

6839 6840 6841 6842 6843 6844
        totalDelay  += pThreadInfo->totalDelay;
        cntDelay   += pThreadInfo->cntDelay;
        if (pThreadInfo->maxDelay > maxDelay) maxDelay = pThreadInfo->maxDelay;
        if (pThreadInfo->minDelay < minDelay) minDelay = pThreadInfo->minDelay;
    }
    cntDelay -= 1;
6845

6846 6847
    if (cntDelay == 0)    cntDelay = 1;
    avgDelay = (double)totalDelay / cntDelay;
6848

6849 6850
    int64_t end = taosGetTimestampMs();
    int64_t t = end - start;
6851

6852
    double tInMs = t/1000.0;
6853

6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886
    if (superTblInfo) {
        fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
                tInMs, superTblInfo->totalInsertRows,
                superTblInfo->totalAffectedRows,
                threads, db_name, superTblInfo->sTblName,
                (tInMs)?
                (double)(superTblInfo->totalInsertRows/tInMs):FLT_MAX);

        if (g_fpOfInsertResult) {
            fprintf(g_fpOfInsertResult,
                    "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
                    tInMs, superTblInfo->totalInsertRows,
                    superTblInfo->totalAffectedRows,
                    threads, db_name, superTblInfo->sTblName,
                    (tInMs)?
                    (double)(superTblInfo->totalInsertRows/tInMs):FLT_MAX);
        }
    } else {
        fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
                tInMs, g_args.totalInsertRows,
                g_args.totalAffectedRows,
                threads, db_name,
                (tInMs)?
                (double)(g_args.totalInsertRows/tInMs):FLT_MAX);
        if (g_fpOfInsertResult) {
            fprintf(g_fpOfInsertResult,
                    "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
                    tInMs, g_args.totalInsertRows,
                    g_args.totalAffectedRows,
                    threads, db_name,
                    (tInMs)?
                    (double)(g_args.totalInsertRows/tInMs):FLT_MAX);
        }
6887
    }
6888 6889 6890

    fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
            avgDelay, maxDelay, minDelay);
6891
    if (g_fpOfInsertResult) {
6892 6893
        fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
                avgDelay, maxDelay, minDelay);
6894
    }
6895

6896
    //taos_close(taos);
6897

6898 6899
    free(pids);
    free(infos);
H
Hui Li 已提交
6900 6901
}

6902
static void *readTable(void *sarg) {
6903
#if 1
6904 6905
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  TAOS *taos = pThreadInfo->taos;
H
hzcheng 已提交
6906
  char command[BUFFER_SIZE] = "\0";
6907 6908 6909
  uint64_t sTime = pThreadInfo->start_time;
  char *tb_prefix = pThreadInfo->tb_prefix;
  FILE *fp = fopen(pThreadInfo->filePath, "a");
H
Hui Li 已提交
6910
  if (NULL == fp) {
6911
    errorPrint( "fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno));
H
Hui Li 已提交
6912 6913
    return NULL;
  }
6914

6915
  int64_t num_of_DPT;
6916 6917
/*  if (pThreadInfo->superTblInfo) {
    num_of_DPT = pThreadInfo->superTblInfo->insertRows; //  nrecords_per_table;
6918
  } else {
6919
  */
6920
      num_of_DPT = g_args.num_of_DPT;
6921
//  }
6922

6923
  int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
S
Shuduo Sang 已提交
6924
  int64_t totalData = num_of_DPT * num_of_tables;
6925
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6926 6927 6928 6929 6930

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
S
Shuduo Sang 已提交
6931
  printf("%"PRId64" records:\n", totalData);
H
hzcheng 已提交
6932 6933
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

6934
  for (int j = 0; j < n; j++) {
H
hzcheng 已提交
6935
    double totalT = 0;
6936
    uint64_t count = 0;
6937 6938
    for (int64_t i = 0; i < num_of_tables; i++) {
      sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64,
6939
              aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
6940

6941
      double t = taosGetTimestampMs();
S
Shuaiqiang Chang 已提交
6942 6943
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
6944

S
Shuaiqiang Chang 已提交
6945
      if (code != 0) {
6946
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6947
        taos_free_result(pSql);
H
hzcheng 已提交
6948
        taos_close(taos);
6949
        fclose(fp);
6950
        return NULL;
H
hzcheng 已提交
6951 6952
      }

6953
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6954 6955 6956
        count++;
      }

6957
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6958 6959
      totalT += t;

S
Shuaiqiang Chang 已提交
6960
      taos_free_result(pSql);
H
hzcheng 已提交
6961 6962
    }

6963
    fprintf(fp, "|%10s  |   %"PRId64"   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
6964
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
6965
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
6966
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000);
H
hzcheng 已提交
6967 6968 6969
  }
  fprintf(fp, "\n");
  fclose(fp);
6970
#endif
H
hzcheng 已提交
6971 6972 6973
  return NULL;
}

6974
static void *readMetric(void *sarg) {
6975
#if 1
6976 6977
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  TAOS *taos = pThreadInfo->taos;
H
hzcheng 已提交
6978
  char command[BUFFER_SIZE] = "\0";
6979
  FILE *fp = fopen(pThreadInfo->filePath, "a");
H
Hui Li 已提交
6980
  if (NULL == fp) {
6981
    printf("fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno));
H
Hui Li 已提交
6982 6983
    return NULL;
  }
6984

6985 6986
  int64_t num_of_DPT = pThreadInfo->superTblInfo->insertRows;
  int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
S
Shuduo Sang 已提交
6987
  int64_t totalData = num_of_DPT * num_of_tables;
6988
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6989 6990 6991 6992 6993

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
S
Shuduo Sang 已提交
6994 6995
  printf("%"PRId64" records:\n", totalData);
  fprintf(fp, "Querying On %"PRId64" records:\n", totalData);
H
hzcheng 已提交
6996 6997

  for (int j = 0; j < n; j++) {
6998
    char condition[COND_BUF_LEN] = "\0";
B
Bomin Zhang 已提交
6999
    char tempS[64] = "\0";
H
hzcheng 已提交
7000

7001
    int64_t m = 10 < num_of_tables ? 10 : num_of_tables;
H
hzcheng 已提交
7002

7003
    for (int64_t i = 1; i <= m; i++) {
H
hzcheng 已提交
7004
      if (i == 1) {
7005
        sprintf(tempS, "t1 = %"PRId64"", i);
H
hzcheng 已提交
7006
      } else {
7007
        sprintf(tempS, " or t1 = %"PRId64" ", i);
H
hzcheng 已提交
7008
      }
7009
      strncat(condition, tempS, COND_BUF_LEN - 1);
H
hzcheng 已提交
7010

L
liu0x54 已提交
7011
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
7012 7013 7014 7015

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

7016
      double t = taosGetTimestampMs();
H
hzcheng 已提交
7017

S
Shuaiqiang Chang 已提交
7018 7019 7020 7021
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
7022
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
7023
        taos_free_result(pSql);
H
hzcheng 已提交
7024
        taos_close(taos);
7025
        fclose(fp);
7026
        return NULL;
H
hzcheng 已提交
7027 7028
      }
      int count = 0;
7029
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
7030 7031
        count++;
      }
7032
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
7033

7034
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
7035 7036
              num_of_tables * num_of_DPT / (t * 1000.0), t);
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0);
H
hzcheng 已提交
7037

S
Shuaiqiang Chang 已提交
7038
      taos_free_result(pSql);
H
hzcheng 已提交
7039 7040 7041 7042
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
7043
#endif
H
hzcheng 已提交
7044 7045 7046
  return NULL;
}

7047 7048 7049 7050 7051 7052 7053
static void prompt()
{
  if (!g_args.answer_yes) {
    printf("         Press enter key to continue or Ctrl-C to stop\n\n");
    (void)getchar();
  }
}
H
Hui Li 已提交
7054

7055
static int insertTestProcess() {
7056

7057 7058 7059
    setupForAnsiEscape();
    int ret = printfInsertMeta();
    resetAfterAnsiEscape();
7060

7061 7062
    if (ret == -1)
        exit(EXIT_FAILURE);
7063

7064 7065 7066 7067 7068 7069
    debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
    g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
    if (NULL == g_fpOfInsertResult) {
        errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
        return -1;
    }
7070

7071 7072
    if (g_fpOfInsertResult)
        printfInsertMetaToFile(g_fpOfInsertResult);
7073

7074
    prompt();
7075

7076
    init_rand_data();
7077

7078 7079 7080 7081 7082 7083
    // create database and super tables
    if(createDatabasesAndStables() != 0) {
        if (g_fpOfInsertResult)
            fclose(g_fpOfInsertResult);
        return -1;
    }
7084

7085 7086 7087 7088 7089 7090
    // pretreatement
    if (prepareSampleData() != 0) {
        if (g_fpOfInsertResult)
            fclose(g_fpOfInsertResult);
        return -1;
    }
7091

7092 7093
    double start;
    double end;
7094

7095 7096 7097 7098
    // create child tables
    start = taosGetTimestampMs();
    createChildTables();
    end = taosGetTimestampMs();
7099

7100 7101 7102 7103 7104 7105 7106 7107
    if (g_totalChildTables > 0) {
        fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
                (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
        if (g_fpOfInsertResult) {
            fprintf(g_fpOfInsertResult,
                    "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
                    (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
        }
7108
    }
7109

7110 7111 7112 7113 7114 7115
    // create sub threads for inserting data
    //start = taosGetTimestampMs();
    for (int i = 0; i < g_Dbs.dbCount; i++) {
        if (g_Dbs.use_metric) {
            if (g_Dbs.db[i].superTblCount > 0) {
                for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
7116

7117
                    SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];
7118

7119 7120 7121 7122 7123 7124 7125 7126 7127 7128
                    if (superTblInfo && (superTblInfo->insertRows > 0)) {
                        startMultiThreadInsertData(
                                g_Dbs.threadCount,
                                g_Dbs.db[i].dbName,
                                g_Dbs.db[i].dbCfg.precision,
                                superTblInfo);
                    }
                }
            }
        } else {
7129
            startMultiThreadInsertData(
7130 7131 7132 7133
                    g_Dbs.threadCount,
                    g_Dbs.db[i].dbName,
                    g_Dbs.db[i].dbCfg.precision,
                    NULL);
7134
        }
H
Hui Li 已提交
7135
    }
7136
    //end = taosGetTimestampMs();
7137

7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148
    //int64_t    totalInsertRows = 0;
    //int64_t    totalAffectedRows = 0;
    //for (int i = 0; i < g_Dbs.dbCount; i++) {
    //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
    //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
    //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
    //}
    //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
    postFreeResource();

    return 0;
7149 7150
}

7151 7152
static void *specifiedTableQuery(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
7153

7154
  if (pThreadInfo->taos == NULL) {
7155 7156
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
7157 7158 7159 7160
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
7161 7162
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
7163
            pThreadInfo->threadID, taos_errstr(NULL));
7164 7165
      return NULL;
    } else {
7166
      pThreadInfo->taos = taos;
7167 7168 7169
    }
  }

7170
  char sqlStr[TSDB_DB_NAME_LEN + 5];
7171
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
7172 7173
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
7174 7175 7176 7177
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
    return NULL;
  }
7178

7179 7180
  uint64_t st = 0;
  uint64_t et = 0;
7181

7182
  uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes;
7183

7184 7185 7186
  uint64_t totalQueried = 0;
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
7187

7188
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') {
7189
    sprintf(pThreadInfo->filePath, "%s-%d",
7190 7191 7192 7193
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }

7194
  while(queryTimes --) {
7195
    if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) <
7196
            (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) {
7197
      taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms
7198 7199
    }

7200 7201 7202
    st = taosGetTimestampMs();

    selectAndGetResult(pThreadInfo,
7203
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]);
7204

7205 7206 7207 7208
    et = taosGetTimestampMs();
    printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n",
              taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0);

7209 7210
    totalQueried ++;
    g_queryInfo.specifiedQueryInfo.totalQueried ++;
7211

7212 7213
    uint64_t  currentPrintTime = taosGetTimestampMs();
    uint64_t  endTs = taosGetTimestampMs();
7214
    if (currentPrintTime - lastPrintTime > 30*1000) {
7215
      debugPrint("%s() LN%d, endTs=%"PRIu64"ms, startTs=%"PRIu64"ms\n",
7216
          __func__, __LINE__, endTs, startTs);
7217
      printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.6f\n",
7218 7219
                    pThreadInfo->threadID,
                    totalQueried,
7220
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
7221
      lastPrintTime = currentPrintTime;
7222
    }
H
Hui Li 已提交
7223
  }
7224 7225
  return NULL;
}
H
Hui Li 已提交
7226

7227
static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) {
7228 7229
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
7230 7231
  sprintf(subTblName, "%s.%s",
          g_queryInfo.dbName,
7232
          g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
7233 7234

  //printf("inSql: %s\n", inSql);
7235

7236 7237
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
7238
    return;
H
Hui Li 已提交
7239
  }
7240

7241
  tstrncpy(outSql, inSql, pos - inSql + 1);
7242
  //printf("1: %s\n", outSql);
7243
  strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1);
7244
  //printf("2: %s\n", outSql);
7245
  strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1);
7246
  //printf("3: %s\n", outSql);
H
Hui Li 已提交
7247 7248
}

7249
static void *superTableQuery(void *sarg) {
7250
  char sqlstr[MAX_QUERY_SQL_LENGTH];
7251
  threadInfo *pThreadInfo = (threadInfo *)sarg;
7252

7253
  if (pThreadInfo->taos == NULL) {
7254 7255 7256 7257 7258 7259 7260 7261
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
7262
            pThreadInfo->threadID, taos_errstr(NULL));
7263 7264
      return NULL;
    } else {
7265
      pThreadInfo->taos = taos;
7266 7267 7268
    }
  }

7269 7270
  uint64_t st = 0;
  uint64_t et = (int64_t)g_queryInfo.superQueryInfo.queryInterval;
7271

7272 7273 7274
  uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes;
  uint64_t totalQueried = 0;
  uint64_t  startTs = taosGetTimestampMs();
7275

7276
  uint64_t  lastPrintTime = taosGetTimestampMs();
7277
  while(queryTimes --) {
7278 7279 7280
    if (g_queryInfo.superQueryInfo.queryInterval
            && (et - st) < (int64_t)g_queryInfo.superQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.superQueryInfo.queryInterval - (et - st)); // ms
7281
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to);
7282
    }
H
Hui Li 已提交
7283

7284
    st = taosGetTimestampMs();
7285
    for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) {
7286
      for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
7287
        memset(sqlstr,0,sizeof(sqlstr));
7288
        replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i);
7289
        if (g_queryInfo.superQueryInfo.result[j][0] != '\0') {
7290
          sprintf(pThreadInfo->filePath, "%s-%d",
7291
                  g_queryInfo.superQueryInfo.result[j],
7292
                  pThreadInfo->threadID);
7293
        }
7294
        selectAndGetResult(pThreadInfo, sqlstr);
7295 7296 7297 7298 7299 7300 7301

        totalQueried++;
        g_queryInfo.superQueryInfo.totalQueried ++;

        int64_t  currentPrintTime = taosGetTimestampMs();
        int64_t  endTs = taosGetTimestampMs();
        if (currentPrintTime - lastPrintTime > 30*1000) {
7302
          printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.3f\n",
7303 7304
                    pThreadInfo->threadID,
                    totalQueried,
7305
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
7306
          lastPrintTime = currentPrintTime;
7307
        }
7308
      }
H
Hui Li 已提交
7309
    }
7310
    et = taosGetTimestampMs();
7311
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%"PRIu64" - %"PRIu64"] once queries duration:%.4fs\n\n",
7312
            taosGetSelfPthreadId(),
7313 7314
            pThreadInfo->start_table_from,
            pThreadInfo->end_table_to,
7315
            (double)(et - st)/1000.0);
7316
  }
7317

7318 7319 7320
  return NULL;
}

7321
static int queryTestProcess() {
7322 7323 7324 7325

  setupForAnsiEscape();
  printfQueryMeta();
  resetAfterAnsiEscape();
7326 7327 7328 7329 7330 7331

  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
7332
          g_queryInfo.port);
7333
  if (taos == NULL) {
7334 7335
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
7336 7337 7338
    exit(-1);
  }

7339
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
7340 7341
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
7342 7343 7344
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
7345
  }
7346

7347
  prompt();
7348

7349 7350 7351
  if (g_args.debug_print || g_args.verbose_print) {
    printfQuerySystemInfo(taos);
  }
7352

7353 7354 7355 7356 7357 7358
  if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
    if (convertHostToServAddr(
        g_queryInfo.host, g_queryInfo.port, &g_queryInfo.serv_addr) != 0)
      exit(-1);
  }

7359 7360 7361
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
7362
  int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent;
7363
  uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount;
7364

7365
  uint64_t startTs = taosGetTimestampMs();
7366

7367 7368
  if ((nSqlCount > 0) && (nConcurrent > 0)) {

7369 7370
    pids  = calloc(1, nConcurrent * nSqlCount * sizeof(pthread_t));
    infos = calloc(1, nConcurrent * nSqlCount * sizeof(threadInfo));
7371 7372

    if ((NULL == pids) || (NULL == infos)) {
7373
      taos_close(taos);
7374
      ERROR_EXIT("memory allocation failed for create threads\n");
7375
    }
7376

7377 7378 7379
    for (uint64_t i = 0; i < nSqlCount; i++) {
        for (int j = 0; j < nConcurrent; j++) {
            uint64_t seq = i * nConcurrent + j;
7380 7381 7382
            threadInfo *pThreadInfo = infos + seq;
            pThreadInfo->threadID = seq;
            pThreadInfo->querySeq = i;
7383 7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396

            if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {

                char sqlStr[MAX_TB_NAME_SIZE*2];
                sprintf(sqlStr, "use %s", g_queryInfo.dbName);
                if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) {
                    taos_close(taos);
                    free(infos);
                    free(pids);
                    errorPrint( "use database %s failed!\n\n",
                        g_queryInfo.dbName);
                    return -1;
                }
            }
7397

7398
            pThreadInfo->taos = NULL;// TODO: workaround to use separate taos connection;
7399

7400
            pthread_create(pids + seq, NULL, specifiedTableQuery,
7401
                pThreadInfo);
7402
        }
7403
    }
7404
  } else {
7405
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
7406
  }
7407

7408 7409
  taos_close(taos);

7410 7411 7412
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
7413 7414
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
7415 7416
    pidsOfSub  = calloc(1, g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t));
    infosOfSub = calloc(1, g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo));
7417 7418

    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
7419 7420
      free(infos);
      free(pids);
7421

7422
      ERROR_EXIT("memory allocation failed for create threads\n");
7423
    }
7424

S
Shuduo Sang 已提交
7425
    int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
7426
    int threads = g_queryInfo.superQueryInfo.threadCnt;
7427

S
Shuduo Sang 已提交
7428
    int64_t a = ntables / threads;
7429 7430 7431 7432
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
7433

S
Shuduo Sang 已提交
7434
    int64_t b = 0;
7435 7436 7437
    if (threads != 0) {
      b = ntables % threads;
    }
7438

7439
    uint64_t tableFrom = 0;
7440
    for (int i = 0; i < threads; i++) {
7441 7442
      threadInfo *pThreadInfo = infosOfSub + i;
      pThreadInfo->threadID = i;
7443

7444
      pThreadInfo->start_table_from = tableFrom;
7445
      pThreadInfo->ntables = i<b?a+1:a;
7446 7447
      pThreadInfo->end_table_to = i < b ? tableFrom + a : tableFrom + a - 1;
      tableFrom = pThreadInfo->end_table_to + 1;
7448 7449
      pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection;
      pthread_create(pidsOfSub + i, NULL, superTableQuery, pThreadInfo);
7450 7451
    }

7452
    g_queryInfo.superQueryInfo.threadCnt = threads;
7453
  } else {
7454
    g_queryInfo.superQueryInfo.threadCnt = 0;
7455
  }
7456

7457 7458 7459 7460 7461 7462
  if ((nSqlCount > 0) && (nConcurrent > 0)) {
    for (int i = 0; i < nConcurrent; i++) {
      for (int j = 0; j < nSqlCount; j++) {
        pthread_join(pids[i * nSqlCount + j], NULL);
      }
    }
7463
  }
S
Shuaiqiang Chang 已提交
7464

7465
  tmfree((char*)pids);
7466
  tmfree((char*)infos);
7467

7468
  for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
7469
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
7470
  }
H
Hui Li 已提交
7471

7472
  tmfree((char*)pidsOfSub);
7473
  tmfree((char*)infosOfSub);
7474

7475
//  taos_close(taos);// TODO: workaround to use separate taos connection;
7476
  uint64_t endTs = taosGetTimestampMs();
7477

7478
  uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried +
7479 7480
    g_queryInfo.superQueryInfo.totalQueried;

7481
  fprintf(stderr, "==== completed total queries: %"PRIu64", the QPS of all threads: %10.3f====\n",
7482
          totalQueried,
7483
          (double)(totalQueried/((endTs-startTs)/1000.0)));
7484 7485 7486
  return 0;
}

7487 7488 7489 7490 7491 7492 7493 7494 7495
static void stable_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
  if (res == NULL || taos_errno(res) != 0) {
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
    return;
  }

  if (param)
7496
    fetchResult(res, (threadInfo *)param);
7497 7498 7499 7500 7501
  // tao_unscribe() will free result.
}

static void specified_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
7502
  if (res == NULL || taos_errno(res) != 0) {
7503 7504
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
7505 7506
    return;
  }
7507

7508
  if (param)
7509
    fetchResult(res, (threadInfo *)param);
7510
  // tao_unscribe() will free result.
H
hzcheng 已提交
7511 7512
}

7513
static TAOS_SUB* subscribeImpl(
7514 7515 7516 7517
        QUERY_CLASS class,
        threadInfo *pThreadInfo,
        char *sql, char* topic, bool restart, uint64_t interval)
{
7518
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
7519

7520 7521 7522 7523
  if ((SPECIFIED_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) {
    tsub = taos_subscribe(
            pThreadInfo->taos,
7524
            restart,
7525
            topic, sql, specified_sub_callback, (void*)pThreadInfo,
7526
            g_queryInfo.specifiedQueryInfo.subscribeInterval);
7527 7528 7529 7530 7531 7532 7533
  } else if ((STABLE_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) {
    tsub = taos_subscribe(
            pThreadInfo->taos,
            restart,
            topic, sql, stable_sub_callback, (void*)pThreadInfo,
            g_queryInfo.superQueryInfo.subscribeInterval);
7534
  } else {
7535 7536
    tsub = taos_subscribe(
            pThreadInfo->taos,
7537
            restart,
7538
            topic, sql, NULL, NULL, interval);
7539
  }
7540

7541
  if (tsub == NULL) {
7542 7543
      errorPrint("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
      return NULL;
7544
  }
7545

7546 7547
  return tsub;
}
H
hzcheng 已提交
7548

7549
static void *superSubscribe(void *sarg) {
7550 7551 7552 7553 7554 7555 7556 7557 7558 7559
    threadInfo *pThreadInfo = (threadInfo *)sarg;
    char subSqlstr[MAX_QUERY_SQL_LENGTH];
    TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT] = {0};
    uint64_t tsubSeq;

    if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
        errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n",
                pThreadInfo->ntables, MAX_QUERY_SQL_COUNT);
        exit(-1);
    }
7560

7561
    if (pThreadInfo->taos == NULL) {
7562 7563 7564 7565 7566 7567 7568 7569 7570 7571
        pThreadInfo->taos = taos_connect(g_queryInfo.host,
                g_queryInfo.user,
                g_queryInfo.password,
                g_queryInfo.dbName,
                g_queryInfo.port);
        if (pThreadInfo->taos == NULL) {
            errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
                    pThreadInfo->threadID, taos_errstr(NULL));
            return NULL;
        }
7572 7573
    }

7574 7575 7576 7577 7578
    char sqlStr[MAX_TB_NAME_SIZE*2];
    sprintf(sqlStr, "use %s", g_queryInfo.dbName);
    if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
        taos_close(pThreadInfo->taos);
        errorPrint( "use database %s failed!\n\n",
7579
                g_queryInfo.dbName);
7580 7581
        return NULL;
    }
7582

7583 7584 7585 7586 7587 7588 7589 7590 7591 7592 7593 7594 7595 7596 7597 7598 7599 7600 7601 7602
    char topic[32] = {0};
    for (uint64_t i = pThreadInfo->start_table_from;
            i <= pThreadInfo->end_table_to; i++) {
        tsubSeq = i - pThreadInfo->start_table_from;
        verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n",
                __func__, __LINE__,
                pThreadInfo->threadID,
                pThreadInfo->start_table_from,
                pThreadInfo->end_table_to, i);
        sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"",
                i, pThreadInfo->querySeq);
        memset(subSqlstr, 0, sizeof(subSqlstr));
        replaceChildTblName(
                g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq],
                subSqlstr, i);
        if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
            sprintf(pThreadInfo->filePath, "%s-%d",
                    g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                    pThreadInfo->threadID);
        }
7603

7604
        verbosePrint("%s() LN%d, [%d] subSqlstr: %s\n",
7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615
                __func__, __LINE__, pThreadInfo->threadID, subSqlstr);
        tsub[tsubSeq] = subscribeImpl(
                STABLE_CLASS,
                pThreadInfo, subSqlstr, topic,
                g_queryInfo.superQueryInfo.subscribeRestart,
                g_queryInfo.superQueryInfo.subscribeInterval);
        if (NULL == tsub[tsubSeq]) {
            taos_close(pThreadInfo->taos);
            return NULL;
        }
    }
7616

7617 7618 7619 7620 7621 7622
    // start loop to consume result
    int consumed[MAX_QUERY_SQL_COUNT];
    for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) {
        consumed[i] = 0;
    }
    TAOS_RES* res = NULL;
7623

7624
    uint64_t st = 0, et = 0;
7625

7626 7627 7628 7629
    while ((g_queryInfo.superQueryInfo.endAfterConsume == -1)
            || (g_queryInfo.superQueryInfo.endAfterConsume >
                consumed[pThreadInfo->end_table_to
                - pThreadInfo->start_table_from])) {
7630

7631 7632 7633 7634 7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645 7646 7647 7648 7649 7650 7651 7652 7653 7654 7655 7656 7657 7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668 7669 7670 7671 7672 7673 7674 7675 7676 7677 7678 7679 7680 7681 7682 7683 7684 7685
        verbosePrint("super endAfterConsume: %d, consumed: %d\n",
                g_queryInfo.superQueryInfo.endAfterConsume,
                consumed[pThreadInfo->end_table_to
                - pThreadInfo->start_table_from]);
        for (uint64_t i = pThreadInfo->start_table_from;
                i <= pThreadInfo->end_table_to; i++) {
            tsubSeq = i - pThreadInfo->start_table_from;
            if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
                continue;
            }

            st = taosGetTimestampMs();
            performancePrint("st: %"PRIu64" et: %"PRIu64" st-et: %"PRIu64"\n", st, et, (st - et));
            res = taos_consume(tsub[tsubSeq]);
            et = taosGetTimestampMs();
            performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st));

            if (res) {
                if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
                    sprintf(pThreadInfo->filePath, "%s-%d",
                            g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                            pThreadInfo->threadID);
                    fetchResult(res, pThreadInfo);
                }
                consumed[tsubSeq] ++;

                if ((g_queryInfo.superQueryInfo.resubAfterConsume != -1)
                        && (consumed[tsubSeq] >=
                            g_queryInfo.superQueryInfo.resubAfterConsume)) {
                    verbosePrint("%s() LN%d, keepProgress:%d, resub super table query: %"PRIu64"\n",
                            __func__, __LINE__,
                            g_queryInfo.superQueryInfo.subscribeKeepProgress,
                            pThreadInfo->querySeq);
                    taos_unsubscribe(tsub[tsubSeq],
                            g_queryInfo.superQueryInfo.subscribeKeepProgress);
                    consumed[tsubSeq]= 0;
                    tsub[tsubSeq] = subscribeImpl(
                            STABLE_CLASS,
                            pThreadInfo, subSqlstr, topic,
                            g_queryInfo.superQueryInfo.subscribeRestart,
                            g_queryInfo.superQueryInfo.subscribeInterval
                            );
                    if (NULL == tsub[tsubSeq]) {
                        taos_close(pThreadInfo->taos);
                        return NULL;
                    }
                }
            }
        }
    }
    verbosePrint("%s() LN%d, super endAfterConsume: %d, consumed: %d\n",
            __func__, __LINE__,
            g_queryInfo.superQueryInfo.endAfterConsume,
            consumed[pThreadInfo->end_table_to - pThreadInfo->start_table_from]);
    taos_free_result(res);
7686

7687 7688
    for (uint64_t i = pThreadInfo->start_table_from;
            i <= pThreadInfo->end_table_to; i++) {
7689 7690
        tsubSeq = i - pThreadInfo->start_table_from;
        taos_unsubscribe(tsub[tsubSeq], 0);
H
hzcheng 已提交
7691
    }
7692

7693 7694
    taos_close(pThreadInfo->taos);
    return NULL;
H
hzcheng 已提交
7695 7696
}

7697 7698
static void *specifiedSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
7699
//  TAOS_SUB*  tsub = NULL;
7700

7701
  if (pThreadInfo->taos == NULL) {
7702
    pThreadInfo->taos = taos_connect(g_queryInfo.host,
7703 7704 7705 7706
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
7707
    if (pThreadInfo->taos == NULL) {
7708
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
7709
            pThreadInfo->threadID, taos_errstr(NULL));
7710 7711 7712 7713
      return NULL;
    }
  }

7714 7715
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
7716 7717
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
7718 7719
    return NULL;
  }
7720

7721 7722 7723 7724
  sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
          "taosdemo-subscribe-%"PRIu64"-%d",
          pThreadInfo->querySeq,
          pThreadInfo->threadID);
7725
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') {
7726
      sprintf(pThreadInfo->filePath, "%s-%d",
7727 7728 7729
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }
7730 7731
  g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl(
          SPECIFIED_CLASS, pThreadInfo,
7732
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
7733
          g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
7734
          g_queryInfo.specifiedQueryInfo.subscribeRestart,
7735
          g_queryInfo.specifiedQueryInfo.subscribeInterval);
7736
  if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) {
7737 7738
      taos_close(pThreadInfo->taos);
      return NULL;
S
Shengliang Guan 已提交
7739
  }
7740

7741
  // start loop to consume result
7742

7743
  g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0;
7744 7745 7746 7747
  while((g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq] == -1)
          || (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] <
              g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq])) {

7748 7749 7750 7751 7752
      printf("consumed[%d]: %d, endAfterConsum[%"PRId64"]: %d\n",
              pThreadInfo->threadID,
              g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID],
              pThreadInfo->querySeq,
              g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq]);
7753
      if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) {
7754
          continue;
7755
      }
7756

7757 7758 7759
      g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] = taos_consume(
              g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]);
      if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) {
7760 7761
          if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0]
                  != 0) {
7762
              sprintf(pThreadInfo->filePath, "%s-%d",
7763 7764 7765
                      g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
          }
7766 7767 7768
          fetchResult(
                  g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID],
                  pThreadInfo);
7769

7770
          g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] ++;
7771
          if ((g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq] != -1)
7772
                && (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] >=
7773 7774 7775 7776
                    g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
              printf("keepProgress:%d, resub specified query: %"PRIu64"\n",
                    g_queryInfo.specifiedQueryInfo.subscribeKeepProgress,
                    pThreadInfo->querySeq);
7777 7778
              g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0;
              taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID],
7779
                      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
7780 7781 7782 7783 7784 7785 7786 7787
              g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] =
                  subscribeImpl(
                          SPECIFIED_CLASS,
                          pThreadInfo,
                          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
                          g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
                          g_queryInfo.specifiedQueryInfo.subscribeRestart,
                          g_queryInfo.specifiedQueryInfo.subscribeInterval);
7788
              if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) {
7789 7790
                  taos_close(pThreadInfo->taos);
                  return NULL;
7791 7792
              }
          }
7793 7794
      }
  }
7795
  taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]);
7796
  taos_close(pThreadInfo->taos);
7797

H
hzcheng 已提交
7798 7799 7800
  return NULL;
}

7801
static int subscribeTestProcess() {
7802
  setupForAnsiEscape();
7803
  printfQueryMeta();
7804
  resetAfterAnsiEscape();
7805

7806
  prompt();
7807

7808 7809 7810 7811 7812
  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
7813
          g_queryInfo.port);
7814
  if (taos == NULL) {
7815 7816
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
7817 7818 7819
    exit(-1);
  }

7820
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
7821 7822
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
7823 7824 7825
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
7826 7827
  }

7828 7829
  taos_close(taos); // TODO: workaround to use separate taos connection;

7830 7831
  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
7832 7833 7834 7835 7836

  pthread_t  *pidsOfStable  = NULL;
  threadInfo *infosOfStable = NULL;

  //==== create threads for query for specified table
S
Shengliang Guan 已提交
7837
  if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) {
7838
    debugPrint("%s() LN%d, sepcified query sqlCount %d.\n",
7839
              __func__, __LINE__,
S
Shengliang Guan 已提交
7840 7841 7842
              g_queryInfo.specifiedQueryInfo.sqlCount);
  } else {
    if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) {
7843
        errorPrint("%s() LN%d, sepcified query sqlCount %d.\n",
S
Shengliang Guan 已提交
7844 7845 7846 7847
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount);
        exit(-1);
    }
7848

7849 7850
    pids  = calloc(
            1,
7851 7852 7853
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(pthread_t));
7854 7855
    infos = calloc(
            1,
7856 7857 7858
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(threadInfo));
S
Shengliang Guan 已提交
7859 7860 7861 7862
    if ((NULL == pids) || (NULL == infos)) {
        errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
        exit(-1);
    }
7863

7864 7865 7866
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
            uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
7867 7868 7869 7870 7871
            threadInfo *pThreadInfo = infos + seq;
            pThreadInfo->threadID = seq;
            pThreadInfo->querySeq = i;
            pThreadInfo->taos = NULL;  // TODO: workaround to use separate taos connection;
            pthread_create(pids + seq, NULL, specifiedSubscribe, pThreadInfo);
7872
        }
S
Shengliang Guan 已提交
7873
    }
H
hzcheng 已提交
7874
  }
7875

7876 7877
  //==== create threads for super table query
  if (g_queryInfo.superQueryInfo.sqlCount <= 0) {
7878
    debugPrint("%s() LN%d, super table query sqlCount %d.\n",
7879 7880 7881 7882
              __func__, __LINE__,
              g_queryInfo.superQueryInfo.sqlCount);
  } else {
    if ((g_queryInfo.superQueryInfo.sqlCount > 0)
7883
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
7884 7885
        pidsOfStable  = calloc(
                1,
7886 7887
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
7888
            sizeof(pthread_t));
7889 7890
        infosOfStable = calloc(
                1,
7891 7892
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
7893
            sizeof(threadInfo));
7894 7895
        if ((NULL == pidsOfStable) || (NULL == infosOfStable)) {
            errorPrint("%s() LN%d, malloc failed for create threads\n",
7896
              __func__, __LINE__);
7897 7898 7899
            // taos_close(taos);
            exit(-1);
        }
7900

7901 7902
        int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
        int threads = g_queryInfo.superQueryInfo.threadCnt;
7903

7904 7905 7906 7907 7908
        int64_t a = ntables / threads;
        if (a < 1) {
            threads = ntables;
            a = 1;
        }
7909

7910 7911 7912 7913
        int64_t b = 0;
        if (threads != 0) {
            b = ntables % threads;
        }
7914

7915
        for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
7916
            uint64_t tableFrom = 0;
7917 7918
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
7919 7920 7921 7922
                threadInfo *pThreadInfo = infosOfStable + seq;
                pThreadInfo->threadID = seq;
                pThreadInfo->querySeq = i;

7923
                pThreadInfo->start_table_from = tableFrom;
7924
                pThreadInfo->ntables = j<b?a+1:a;
7925 7926
                pThreadInfo->end_table_to = j<b?tableFrom+a:tableFrom+a-1;
                tableFrom = pThreadInfo->end_table_to + 1;
7927
                pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection;
7928
                pthread_create(pidsOfStable + seq,
7929
                        NULL, superSubscribe, pThreadInfo);
7930 7931
            }
        }
7932

7933
        g_queryInfo.superQueryInfo.threadCnt = threads;
7934

7935 7936 7937 7938 7939 7940
        for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                pthread_join(pidsOfStable[seq], NULL);
            }
        }
7941
    }
H
hzcheng 已提交
7942
  }
7943

7944 7945 7946 7947 7948
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
    for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
        uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
        pthread_join(pids[seq], NULL);
    }
7949
  }
H
hzcheng 已提交
7950

7951
  tmfree((char*)pids);
7952
  tmfree((char*)infos);
H
hzcheng 已提交
7953

7954 7955
  tmfree((char*)pidsOfStable);
  tmfree((char*)infosOfStable);
7956
//   taos_close(taos);
7957
  return 0;
H
hzcheng 已提交
7958 7959
}

7960
static void initOfInsertMeta() {
7961
  memset(&g_Dbs, 0, sizeof(SDbs));
7962

7963
  // set default values
7964
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7965
  g_Dbs.port = 6030;
7966 7967
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
7968
  g_Dbs.threadCount = 2;
7969 7970

  g_Dbs.use_metric = g_args.use_metric;
L
Liu Tao 已提交
7971 7972
}

7973
static void initOfQueryMeta() {
7974
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
7975

7976
  // set default values
7977
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7978
  g_queryInfo.port = 6030;
7979 7980
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
L
Liu Tao 已提交
7981 7982
}

7983
static void setParaFromArg(){
7984 7985 7986 7987 7988
    if (g_args.host) {
        tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE);
    } else {
        tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
    }
7989

7990 7991 7992
    if (g_args.user) {
        tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE);
    }
7993

7994 7995 7996
    if (g_args.password) {
        tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE);
    }
L
Liu Tao 已提交
7997

7998 7999 8000
    if (g_args.port) {
        g_Dbs.port = g_args.port;
    }
8001

8002 8003
    g_Dbs.threadCount = g_args.num_of_threads;
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
8004

8005 8006
    g_Dbs.dbCount = 1;
    g_Dbs.db[0].drop = true;
L
Liu Tao 已提交
8007

8008 8009 8010
    tstrncpy(g_Dbs.db[0].dbName, g_args.database, TSDB_DB_NAME_LEN);
    g_Dbs.db[0].dbCfg.replica = g_args.replica;
    tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", 8);
8011

8012
    tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
8013

8014 8015
    g_Dbs.use_metric = g_args.use_metric;
    g_Dbs.insert_only = g_args.insert_only;
L
Liu Tao 已提交
8016

8017
    g_Dbs.do_aggreFunc = true;
8018

8019 8020
    char dataString[STRING_LEN];
    char **data_type = g_args.datatype;
L
Liu Tao 已提交
8021

8022
    memset(dataString, 0, STRING_LEN);
H
hzcheng 已提交
8023

8024 8025 8026 8027
    if (strcasecmp(data_type[0], "BINARY") == 0
            || strcasecmp(data_type[0], "BOOL") == 0
            || strcasecmp(data_type[0], "NCHAR") == 0 ) {
        g_Dbs.do_aggreFunc = false;
8028
    }
8029

8030 8031 8032 8033 8034 8035 8036
    if (g_args.use_metric) {
        g_Dbs.db[0].superTblCount = 1;
        tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", TSDB_TABLE_NAME_LEN);
        g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
        g_Dbs.threadCount = g_args.num_of_threads;
        g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
        g_Dbs.asyncMode = g_args.async_mode;
8037

8038 8039 8040 8041 8042 8043 8044
        g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
        g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
        g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
        g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
        tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix,
                g_args.tb_prefix, TSDB_TABLE_NAME_LEN - 20);
        tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
8045

8046 8047 8048 8049 8050 8051 8052 8053
        if (g_args.iface == INTERFACE_BUT) {
            g_Dbs.db[0].superTbls[0].iface = TAOSC_IFACE;
        } else {
            g_Dbs.db[0].superTbls[0].iface = g_args.iface;
        }
        tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp,
                "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
        g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
8054

8055 8056
        g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
        g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len;
8057

8058 8059 8060 8061 8062
        g_Dbs.db[0].superTbls[0].columnCount = 0;
        for (int i = 0; i < MAX_NUM_COLUMNS; i++) {
            if (data_type[i] == NULL) {
                break;
            }
8063

8064 8065 8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081 8082 8083 8084 8085 8086 8087 8088 8089 8090 8091 8092 8093
            tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
                    data_type[i], strlen(data_type[i]) + 1);
            g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;
            g_Dbs.db[0].superTbls[0].columnCount++;
        }

        if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
            g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
        } else {
            for (int i = g_Dbs.db[0].superTbls[0].columnCount;
                    i < g_args.num_of_CPR; i++) {
                tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
                        "INT", strlen("INT") + 1);
                g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;
                g_Dbs.db[0].superTbls[0].columnCount++;
            }
        }

        tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType,
                "INT", strlen("INT") + 1);
        g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;

        tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType,
                "BINARY", strlen("BINARY") + 1);
        g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;
        g_Dbs.db[0].superTbls[0].tagCount = 2;
    } else {
        g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
        g_Dbs.db[0].superTbls[0].tagCount = 0;
    }
8094 8095 8096 8097 8098 8099
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
8100

8101 8102 8103
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
8104 8105 8106
    exit(-1);
  }

8107 8108 8109 8110 8111 8112 8113 8114 8115 8116 8117 8118 8119 8120 8121 8122 8123 8124 8125 8126 8127 8128 8129 8130
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

8131
static void querySqlFile(TAOS* taos, char* sqlFile)
8132 8133 8134 8135 8136 8137
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
8138

8139 8140 8141 8142 8143 8144
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

8145
  double t = taosGetTimestampMs();
8146

8147
  while((read_len = tgetline(&line, &line_len, fp)) != -1) {
8148 8149 8150 8151 8152 8153 8154 8155 8156 8157 8158 8159
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
8160
    }
8161 8162

    memcpy(cmd + cmd_len, line, read_len);
8163 8164 8165
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) {
        errorPrint("%s() LN%d, queryDbExec %s failed!\n",
               __func__, __LINE__, cmd);
8166 8167 8168 8169 8170
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
8171 8172
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
8173 8174
  }

8175
  t = taosGetTimestampMs() - t;
8176
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
8177

8178 8179 8180 8181
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
8182 8183
}

8184
static void testMetaFile() {
8185
    if (INSERT_TEST == g_args.test_mode) {
8186 8187
      if (g_Dbs.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
8188

8189
      insertTestProcess();
8190

8191
    } else if (QUERY_TEST == g_args.test_mode) {
8192 8193
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
8194

8195
      queryTestProcess();
8196

8197
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
8198 8199
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
8200

8201
      subscribeTestProcess();
8202

8203 8204 8205
    }  else {
      ;
    }
8206
}
8207

8208
static void queryResult() {
8209
  // query data
8210

8211
  pthread_t read_id;
8212
  threadInfo *pThreadInfo = calloc(1, sizeof(threadInfo));
8213 8214 8215
  assert(pThreadInfo);
  pThreadInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
  pThreadInfo->start_table_from = 0;
8216

8217
  //pThreadInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
8218
  if (g_args.use_metric) {
8219 8220 8221 8222
    pThreadInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
    pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
    pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
    tstrncpy(pThreadInfo->tb_prefix,
8223
          g_Dbs.db[0].superTbls[0].childTblPrefix, TSDB_TABLE_NAME_LEN - 20);
8224
  } else {
8225 8226
    pThreadInfo->ntables = g_args.num_of_tables;
    pThreadInfo->end_table_to = g_args.num_of_tables -1;
8227
    tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, TSDB_TABLE_NAME_LEN);
8228 8229
  }

8230
  pThreadInfo->taos = taos_connect(
8231 8232 8233 8234 8235
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
          g_Dbs.port);
8236
  if (pThreadInfo->taos == NULL) {
8237 8238
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
8239
    free(pThreadInfo);
8240 8241 8242
    exit(-1);
  }

8243
  tstrncpy(pThreadInfo->filePath, g_Dbs.resultFile, MAX_FILE_NAME_LEN);
8244 8245

  if (!g_Dbs.use_metric) {
8246
    pthread_create(&read_id, NULL, readTable, pThreadInfo);
8247
  } else {
8248
    pthread_create(&read_id, NULL, readMetric, pThreadInfo);
8249 8250
  }
  pthread_join(read_id, NULL);
8251 8252
  taos_close(pThreadInfo->taos);
  free(pThreadInfo);
8253 8254
}

8255 8256
static void testCmdLine() {

8257 8258 8259 8260 8261 8262 8263 8264 8265 8266
  if (strlen(configDir)) {
    wordexp_t full_path;
    if (wordexp(configDir, &full_path, 0) != 0) {
      errorPrint( "Invalid path %s\n", configDir);
      return;
    }
    taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
    wordfree(&full_path);
  }

8267 8268
  g_args.test_mode = INSERT_TEST;
  insertTestProcess();
8269

8270
  if (false == g_Dbs.insert_only)
8271
    queryResult();
8272 8273
}

8274 8275 8276
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

8277
  debugPrint("meta file: %s\n", g_args.metaFile);
8278 8279 8280

  if (g_args.metaFile) {
    initOfInsertMeta();
8281
    initOfQueryMeta();
8282 8283 8284 8285 8286 8287 8288 8289 8290 8291 8292 8293 8294

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
8295 8296 8297 8298
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
8299
          g_Dbs.port);
8300
      querySqlFile(qtaos, g_args.sqlFile);
8301 8302 8303 8304 8305
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
8306 8307 8308

    if (g_dupstr)
        free(g_dupstr);
H
hzcheng 已提交
8309
  }
8310 8311

  return 0;
H
hzcheng 已提交
8312
}
8313