taosdemo.c 245.5 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

21
#include <stdint.h>
S
slguan 已提交
22
#define _GNU_SOURCE
23 24
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
41
#else
S
TD-1057  
Shengliang Guan 已提交
42 43
  #include <regex.h>
  #include <stdio.h>
44
#endif
H
hzcheng 已提交
45

46
#include <assert.h>
47
#include <stdlib.h>
48 49
#include "cJSON.h"

50
#include "os.h"
H
hzcheng 已提交
51
#include "taos.h"
52
#include "taoserror.h"
H
Hui Li 已提交
53
#include "tutil.h"
S
slguan 已提交
54

55 56 57
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
58 59
extern char configDir[];

60 61 62 63
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

64 65 66 67 68 69
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};
70

71 72
#define MAX_RECORDS_PER_REQ     32766

73 74
#define HEAD_BUFF_LEN    1024*24  // 16*1024 + (192+32)*2 + insert into ..

75 76
#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
77
#define COND_BUF_LEN        BUFFER_SIZE - 30
78 79
#define MAX_USERNAME_SIZE  64
#define MAX_PASSWORD_SIZE  64
80
#define MAX_DB_NAME_SIZE   64
81
#define MAX_HOSTNAME_SIZE  64
82
#define MAX_TB_NAME_SIZE   64
83
#define MAX_DATA_SIZE      (16*1024)+20     // max record len: 16*1024, timestamp string and ,('') need extra space
84 85 86 87 88 89 90 91 92 93
#define MAX_NUM_DATATYPE   10
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
#define MAX_FILE_NAME_LEN  256

#define   MAX_SAMPLES_ONCE_FROM_FILE   10000
#define   MAX_NUM_DATATYPE 10

#define   MAX_DB_COUNT           8
94
#define   MAX_SUPER_TABLE_COUNT  200
95 96 97
#define   MAX_COLUMN_COUNT       1024
#define   MAX_TAG_COUNT          128

98
#define   MAX_QUERY_SQL_COUNT    100
99
#define   MAX_QUERY_SQL_LENGTH   1024
100 101

#define   MAX_DATABASE_COUNT     256
102
#define INPUT_BUF_LEN   256
103

104
#define DEFAULT_TIMESTAMP_STEP  1
105

106

107 108 109 110 111
typedef enum CREATE_SUB_TALBE_MOD_EN {
  PRE_CREATE_SUBTBL,
  AUTO_CREATE_SUBTBL,
  NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
112

113 114
typedef enum TALBE_EXISTS_EN {
  TBL_NO_EXISTS,
115
  TBL_ALREADY_EXISTS,
116 117 118
  TBL_EXISTS_BUTT
} TALBE_EXISTS_EN;

119
enum enumSYNC_MODE {
120 121
  SYNC_MODE,
  ASYNC_MODE,
122 123
  MODE_BUT
};
124

S
Shuduo Sang 已提交
125
enum enum_TAOS_INTERFACE {
126 127 128
    TAOSC_IFACE,
    REST_IFACE,
    STMT_IFACE,
129 130 131
    INTERFACE_BUT
};

132 133 134 135 136 137
typedef enum enumQUERY_CLASS {
    SPECIFIED_CLASS,
    STABLE_CLASS,
    CLASS_BUT
} QUERY_CLASS;

S
Shuduo Sang 已提交
138
typedef enum enum_PROGRESSIVE_OR_INTERLACE {
139 140 141
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
142
} PROG_OR_INTERLACE_MODE;
143

144
typedef enum enumQUERY_TYPE {
145
  NO_INSERT_TYPE,
146
  INSERT_TYPE,
147
  QUERY_TYPE_BUT
148
} QUERY_TYPE;
149 150 151 152 153 154 155

enum _show_db_index {
  TSDB_SHOW_DB_NAME_INDEX,
  TSDB_SHOW_DB_CREATED_TIME_INDEX,
  TSDB_SHOW_DB_NTABLES_INDEX,
  TSDB_SHOW_DB_VGROUPS_INDEX,
  TSDB_SHOW_DB_REPLICA_INDEX,
156
  TSDB_SHOW_DB_QUORUM_INDEX,
157 158 159 160 161 162 163 164 165 166
  TSDB_SHOW_DB_DAYS_INDEX,
  TSDB_SHOW_DB_KEEP_INDEX,
  TSDB_SHOW_DB_CACHE_INDEX,
  TSDB_SHOW_DB_BLOCKS_INDEX,
  TSDB_SHOW_DB_MINROWS_INDEX,
  TSDB_SHOW_DB_MAXROWS_INDEX,
  TSDB_SHOW_DB_WALLEVEL_INDEX,
  TSDB_SHOW_DB_FSYNC_INDEX,
  TSDB_SHOW_DB_COMP_INDEX,
  TSDB_SHOW_DB_CACHELAST_INDEX,
167
  TSDB_SHOW_DB_PRECISION_INDEX,
168 169 170 171 172 173 174 175 176 177
  TSDB_SHOW_DB_UPDATE_INDEX,
  TSDB_SHOW_DB_STATUS_INDEX,
  TSDB_MAX_SHOW_DB
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
  TSDB_SHOW_STABLES_NAME_INDEX,
  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
  TSDB_SHOW_STABLES_COLUMNS_INDEX,
178 179
  TSDB_SHOW_STABLES_METRIC_INDEX,
  TSDB_SHOW_STABLES_UID_INDEX,
180
  TSDB_SHOW_STABLES_TID_INDEX,
181
  TSDB_SHOW_STABLES_VGID_INDEX,
182
  TSDB_MAX_SHOW_STABLES
183 184
};

185 186 187 188 189 190 191 192 193 194 195 196 197 198
enum _describe_table_index {
  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
  TSDB_MAX_DESCRIBE_METRIC
};

typedef struct {
  char field[TSDB_COL_NAME_LEN + 1];
  char type[16];
  int length;
  char note[128];
} SColDes;
H
hzcheng 已提交
199 200

/* Used by main to communicate with parse_opt. */
201 202
static char *g_dupstr = NULL;

203 204
typedef struct SArguments_S {
  char *   metaFile;
205
  uint32_t test_mode;
206 207
  char *   host;
  uint16_t port;
S
Shuduo Sang 已提交
208
  uint16_t iface;
209 210 211 212 213 214 215
  char *   user;
  char *   password;
  char *   database;
  int      replica;
  char *   tb_prefix;
  char *   sqlFile;
  bool     use_metric;
216
  bool     drop_database;
217
  bool     insert_only;
218
  bool     answer_yes;
219
  bool     debug_print;
220
  bool     verbose_print;
221
  bool     performance_print;
222
  char *   output_file;
223
  bool     async_mode;
224
  char *   datatype[MAX_NUM_DATATYPE + 1];
225 226 227 228
  uint32_t len_of_binary;
  uint32_t num_of_CPR;
  uint32_t num_of_threads;
  uint64_t insert_interval;
229
  int64_t  query_times;
230 231 232
  uint64_t interlace_rows;
  uint64_t num_of_RPR;                  // num_of_records_per_req
  uint64_t max_sql_len;
233 234
  int64_t  num_of_tables;
  int64_t  num_of_DPT;
235
  int      abort;
236 237
  int      disorderRatio;               // 0: no disorder, >0: x%
  int      disorderRange;               // ms or us by database precision
238
  uint32_t method_of_delete;
239
  char **  arg_list;
240 241
  uint64_t totalInsertRows;
  uint64_t totalAffectedRows;
242 243 244
} SArguments;

typedef struct SColumn_S {
245 246 247
  char      field[TSDB_COL_NAME_LEN + 1];
  char      dataType[MAX_TB_NAME_SIZE];
  uint32_t  dataLen;
248
  char  note[128];
249 250 251
} StrColumn;

typedef struct SSuperTable_S {
252
  char         sTblName[MAX_TB_NAME_SIZE+1];
253
  int64_t      childTblCount;
254
  bool         childTblExists;          // 0: no, 1: yes
255 256
  uint64_t     batchCreateTableNum;     // 0: no batch,  > 0: batch table number in one sql
  uint8_t      autoCreateTable;         // 0: create sub table, 1: auto create sub table
257
  char         childTblPrefix[MAX_TB_NAME_SIZE];
258
  char         dataSource[MAX_TB_NAME_SIZE+1];  // rand_gen or sample
259
  uint16_t     insertMode;              // 0: taosc, 1: rest, 2: stmt
260
  int64_t      childTblLimit;
261
  uint64_t     childTblOffset;
262

263
//  int          multiThreadWriteOneTbl;  // 0: no, 1: yes
264
  uint64_t     interlaceRows;           //
265 266
  int          disorderRatio;           // 0: no disorder, >0: x%
  int          disorderRange;           // ms or us by database precision
267
  uint64_t     maxSqlLen;               //
268

269
  uint64_t     insertInterval;          // insert interval, will override global insert interval
270
  int64_t      insertRows;
271
  int64_t      timeStampStep;
272
  char         startTimestamp[MAX_TB_NAME_SIZE];
273
  char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
274 275
  char         sampleFile[MAX_FILE_NAME_LEN+1];
  char         tagsFile[MAX_FILE_NAME_LEN+1];
276

277
  uint32_t     columnCount;
278
  StrColumn    columns[MAX_COLUMN_COUNT];
279
  uint32_t     tagCount;
280 281 282
  StrColumn    tags[MAX_TAG_COUNT];

  char*        childTblName;
283
  char*        colsOfCreateChildTable;
284 285
  uint64_t     lenOfOneRow;
  uint64_t     lenOfTagOfOneRow;
286 287 288 289 290

  char*        sampleDataBuf;
  //int          sampleRowCount;
  //int          sampleUsePos;

291
  uint32_t     tagSource;    // 0: rand, 1: tag sample
292
  char*        tagDataBuf;
293 294
  uint32_t     tagSampleCount;
  uint32_t     tagUsePos;
295

296
  // statistics
297 298
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
299 300 301 302 303
} SSuperTable;

typedef struct {
  char     name[TSDB_DB_NAME_LEN + 1];
  char     create_time[32];
S
Shuduo Sang 已提交
304
  int64_t  ntables;
305
  int32_t  vgroups;
306 307
  int16_t  replica;
  int16_t  quorum;
308
  int16_t  days;
309 310 311 312 313 314 315 316 317 318 319 320 321 322
  char     keeplist[32];
  int32_t  cache; //MB
  int32_t  blocks;
  int32_t  minrows;
  int32_t  maxrows;
  int8_t   wallevel;
  int32_t  fsync;
  int8_t   comp;
  int8_t   cachelast;
  char     precision[8];   // time resolution
  int8_t   update;
  char     status[16];
} SDbInfo;

323
typedef struct SDbCfg_S {
324
//  int       maxtablesPerVnode;
325 326
  uint32_t  minRows;        // 0 means default
  uint32_t  maxRows;        // 0 means default
327 328
  int       comp;
  int       walLevel;
329
  int       cacheLast;
330
  int       fsync;
331 332 333 334 335 336 337
  int       replica;
  int       update;
  int       keep;
  int       days;
  int       cache;
  int       blocks;
  int       quorum;
338
  char      precision[MAX_TB_NAME_SIZE];
339 340 341 342
} SDbCfg;

typedef struct SDataBase_S {
  char         dbName[MAX_DB_NAME_SIZE];
343
  bool         drop;  // 0: use exists, 1: if exists, drop then new create
344
  SDbCfg       dbCfg;
345
  uint64_t     superTblCount;
346 347 348 349
  SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
} SDataBase;

typedef struct SDbs_S {
350
  char         cfgDir[MAX_FILE_NAME_LEN+1];
351
  char         host[MAX_HOSTNAME_SIZE];
352 353
  struct sockaddr_in serv_addr;

354
  uint16_t     port;
355 356
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
357
  char         resultFile[MAX_FILE_NAME_LEN+1];
358 359 360
  bool         use_metric;
  bool         insert_only;
  bool         do_aggreFunc;
361
  bool         asyncMode;
362

363 364 365
  uint32_t     threadCount;
  uint32_t     threadCountByCreateTbl;
  uint32_t     dbCount;
366 367 368
  SDataBase    db[MAX_DB_COUNT];

  // statistics
369 370
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
371

372 373
} SDbs;

374
typedef struct SpecifiedQueryInfo_S {
375 376 377
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint64_t     concurrent;
  uint64_t     sqlCount;
378
  uint32_t     asyncMode; // 0: sync, 1: async
379 380
  uint64_t     subscribeInterval; // ms
  uint64_t     queryTimes;
381 382
  int          subscribeRestart;
  int          subscribeKeepProgress;
383
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
384
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
385
  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
386
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
387
  uint64_t     totalQueried;
388
} SpecifiedQueryInfo;
389

390
typedef struct SuperQueryInfo_S {
391
  char         sTblName[MAX_TB_NAME_SIZE+1];
392 393
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint32_t     threadCnt;
394
  uint32_t     asyncMode; // 0: sync, 1: async
395
  uint64_t     subscribeInterval; // ms
396 397
  int          subscribeRestart;
  int          subscribeKeepProgress;
398
  uint64_t     queryTimes;
399
  int64_t      childTblCount;
400
  char         childTblPrefix[MAX_TB_NAME_SIZE];
401
  uint64_t     sqlCount;
402
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
403
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
404
  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
405
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
406

407
  char*        childTblName;
408
  uint64_t     totalQueried;
409
} SuperQueryInfo;
410 411

typedef struct SQueryMetaInfo_S {
412
  char         cfgDir[MAX_FILE_NAME_LEN+1];
413
  char         host[MAX_HOSTNAME_SIZE];
414
  uint16_t     port;
415
  struct       sockaddr_in serv_addr;
416 417
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
418
  char         dbName[MAX_DB_NAME_SIZE+1];
419
  char         queryMode[MAX_TB_NAME_SIZE];  // taosc, rest
420

421
  SpecifiedQueryInfo  specifiedQueryInfo;
422
  SuperQueryInfo      superQueryInfo;
423
  uint64_t     totalQueried;
424 425 426
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
427
  TAOS *    taos;
428
  TAOS_STMT *stmt;
429 430 431 432 433 434 435
  int       threadID;
  char      db_name[MAX_DB_NAME_SIZE+1];
  uint32_t  time_precision;
  char      fp[4096];
  char      tb_prefix[MAX_TB_NAME_SIZE];
  uint64_t  start_table_from;
  uint64_t  end_table_to;
S
Shuduo Sang 已提交
436
  int64_t   ntables;
437 438 439 440
  uint64_t  data_of_rate;
  int64_t   start_time;
  char*     cols;
  bool      use_metric;
441
  SSuperTable* superTblInfo;
S
Shuduo Sang 已提交
442
  char      *buffer;    // sql cmd buffer
443 444

  // for async insert
445 446
  tsem_t    lock_sem;
  int64_t   counter;
447 448
  uint64_t  st;
  uint64_t  et;
449
  uint64_t  lastTs;
450

451
  // sample data
452
  int64_t   samplePos;
453
  // statistics
454 455
  uint64_t  totalInsertRows;
  uint64_t  totalAffectedRows;
456 457

  // insert delay statistics
458 459 460 461 462
  uint64_t  cntDelay;
  uint64_t  totalDelay;
  uint64_t  avgDelay;
  uint64_t  maxDelay;
  uint64_t  minDelay;
463

464
  // seq of query or subscribe
465
  uint64_t  querySeq;   // sequence number of sql command
466

467 468
} threadInfo;

469
#ifdef WINDOWS
470 471
#define _CRT_RAND_S

472
#include <windows.h>
473 474 475 476 477
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
478 479
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
480 481
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
482 483 484 485

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

486
static void setupForAnsiEscape(void) {
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
505
  }
506 507
}

508
static void resetAfterAnsiEscape(void) {
509
  // Reset colors
510
  printf("\x1b[0m");
511 512 513 514 515 516

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
517

518
static int taosRandom()
519 520 521 522 523 524
{
    int number;
    rand_s(&number);

    return number;
}
525
#else   // Not windows
526
static void setupForAnsiEscape(void) {}
527

528
static void resetAfterAnsiEscape(void) {
529 530 531
  // Reset colors
  printf("\x1b[0m");
}
532

533 534
#include <time.h>

535
static int taosRandom()
536
{
537
  return rand();
538 539
}

540
#endif // ifdef Windows
541

542
static void prompt();
543
static int createDatabasesAndStables();
544
static void createChildTables();
545
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet);
546 547
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
        char* sqlstr, char *resultFile);
548 549 550 551 552 553 554

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
555
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)",
556 557
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

558
SArguments g_args = {
559 560 561 562
                     NULL,            // metaFile
                     0,               // test_mode
                     "127.0.0.1",     // host
                     6030,            // port
S
Shuduo Sang 已提交
563
                     TAOSC_IFACE,     // iface
564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595
                     "root",          // user
                     #ifdef _TD_POWER_
                     "powerdb",      // password
                     #else
                     "taosdata",      // password
                     #endif
                     "test",          // database
                     1,               // replica
                     "t",             // tb_prefix
                     NULL,            // sqlFile
                     true,            // use_metric
                     true,            // drop_database
                     true,            // insert_only
                     false,           // debug_print
                     false,           // verbose_print
                     false,           // performance statistic print
                     false,           // answer_yes;
                     "./output.txt",  // output_file
                     0,               // mode : sync or async
                     {
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     },
                     16,              // len_of_binary
                     4,               // num_of_CPR
                     10,              // num_of_connections/thread
                     0,               // insert_interval
                     1,               // query_times
                     0,               // interlace_rows;
                     30000,           // num_of_RPR
sangshuduo's avatar
sangshuduo 已提交
596
                     (1024*1024),         // max_sql_len
597 598 599 600 601 602 603
                     10000,           // num_of_tables
                     10000,           // num_of_DPT
                     0,               // abort
                     0,               // disorderRatio
                     1000,            // disorderRange
                     1,               // method_of_delete
                     NULL             // arg_list
604 605 606 607 608
};



static SDbs            g_Dbs;
609
static int64_t         g_totalChildTables = 0;
610 611 612 613
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

#define debugPrint(fmt, ...) \
614
    do { if (g_args.debug_print || g_args.verbose_print) \
615
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
616

617
#define verbosePrint(fmt, ...) \
618 619
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
620

621 622 623 624
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)

625 626 627 628
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)


629
///////////////////////////////////////////////////
H
hzcheng 已提交
630

631 632
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

633 634 635 636 637 638 639 640
#ifndef TAOSDEMO_COMMIT_SHA1
#define TAOSDEMO_COMMIT_SHA1 "unknown"
#endif

#ifndef TD_VERNUMBER
#define TD_VERNUMBER    "unknown"
#endif

641
#ifndef TAOSDEMO_STATUS
642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658
#define TAOSDEMO_STATUS "unknown"
#endif

static void printVersion() {
    char tdengine_ver[] = TD_VERNUMBER;
    char taosdemo_ver[] = TAOSDEMO_COMMIT_SHA1;
    char taosdemo_status[] = TAOSDEMO_STATUS;

    if (strlen(taosdemo_status) == 0) {
        printf("taosdemo verison %s-%s\n",
                tdengine_ver, taosdemo_ver);
    } else {
        printf("taosdemo verison %s-%s, status:%s\n",
                tdengine_ver, taosdemo_ver, taosdemo_status);
    }
}

659
static void printHelp() {
660
  char indent[10] = "        ";
661
  printf("%s%s%s%s\n", indent, "-f", indent,
662
          "The meta file to the execution procedure. Default is './meta.json'.");
663
  printf("%s%s%s%s\n", indent, "-u", indent,
664 665
          "The TDengine user name to use when connecting to the server. Default is 'root'.");
#ifdef _TD_POWER_
666
  printf("%s%s%s%s\n", indent, "-P", indent,
667
          "The password to use when connecting to the server. Default is 'powerdb'.");
668
  printf("%s%s%s%s\n", indent, "-c", indent,
669 670
          "Configuration directory. Default is '/etc/power/'.");
#else
671
  printf("%s%s%s%s\n", indent, "-P", indent,
672
          "The password to use when connecting to the server. Default is 'taosdata'.");
673
  printf("%s%s%s%s\n", indent, "-c", indent,
674
          "Configuration directory. Default is '/etc/taos/'.");
675
#endif
676
  printf("%s%s%s%s\n", indent, "-h", indent,
677
          "The host to connect to TDengine. Default is localhost.");
678
  printf("%s%s%s%s\n", indent, "-p", indent,
679
          "The TCP/IP port number to use for the connection. Default is 0.");
680 681
  printf("%s%s%s%s\n", indent, "-I", indent,
          "The interface (taosc, rest, and stmt) taosdemo uses. Default is 'taosc'.");
682
  printf("%s%s%s%s\n", indent, "-d", indent,
683
          "Destination database. Default is 'test'.");
684
  printf("%s%s%s%s\n", indent, "-a", indent,
685
          "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
686
  printf("%s%s%s%s\n", indent, "-m", indent,
687 688
          "Table prefix name. Default is 't'.");
  printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
689
  printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
690
  printf("%s%s%s%s\n", indent, "-o", indent,
691
          "Direct output to the named file. Default is './output.txt'.");
692
  printf("%s%s%s%s\n", indent, "-q", indent,
693
          "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC.");
694
  printf("%s%s%s%s\n", indent, "-b", indent,
sangshuduo's avatar
sangshuduo 已提交
695
          "The data_type of columns, default: INT,INT,INT,INT.");
696
  printf("%s%s%s%s\n", indent, "-w", indent,
697
          "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
698 699 700
  printf("%s%s%s%s%d\n", indent, "-l", indent,
          "The number of columns per record. Default is 4. Max values is ",
       MAX_NUM_DATATYPE);
701
  printf("%s%s%s%s\n", indent, "-T", indent,
702
          "The number of threads. Default is 10.");
703
  printf("%s%s%s%s\n", indent, "-i", indent,
704
          "The sleep time (ms) between insertion. Default is 0.");
705
  printf("%s%s%s%s\n", indent, "-r", indent,
706
          "The number of records per request. Default is 30000.");
707
  printf("%s%s%s%s\n", indent, "-t", indent,
708
          "The number of tables. Default is 10000.");
709
  printf("%s%s%s%s\n", indent, "-n", indent,
710 711 712
          "The number of records per table. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
  printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
713
  printf("%s%s%s%s\n", indent, "-O", indent,
714
          "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order.");
715
  printf("%s%s%s%s\n", indent, "-R", indent,
716
          "Out of order data's range, ms, default is 1000.");
717
  printf("%s%s%s%s\n", indent, "-g", indent,
718
          "Print debug info.");
719
  printf("%s%s%s\n", indent, "-V, --version\t",
720
          "Print version info.");
721 722
  printf("%s%s%s%s\n", indent, "--help\t", indent,
          "Print command line arguments list info.");
723
/*    printf("%s%s%s%s\n", indent, "-D", indent,
724 725 726 727
          "if elete database if exists. 0: no, 1: yes, default is 1");
          */
}

728 729 730 731 732 733 734 735 736 737 738 739 740 741 742
static bool isStringNumber(char *input)
{
  int len = strlen(input);
  if (0 == len) {
    return false;
  }

  for (int i = 0; i < len; i++) {
    if (!isdigit(input[i]))
      return false;
  }

  return true;
}

743
static void parse_args(int argc, char *argv[], SArguments *arguments) {
744 745 746 747 748

  for (int i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-f") == 0) {
      arguments->metaFile = argv[++i];
    } else if (strcmp(argv[i], "-c") == 0) {
H
Haojun Liao 已提交
749 750 751 752 753
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-c need a valid path following!\n");
        exit(EXIT_FAILURE);
      }
754
      tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN);
755
    } else if (strcmp(argv[i], "-h") == 0) {
H
Haojun Liao 已提交
756 757 758 759 760
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-h need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
761 762
      arguments->host = argv[++i];
    } else if (strcmp(argv[i], "-p") == 0) {
763
      if ((argc == i+1) ||
764
        (!isStringNumber(argv[i+1]))) {
765 766 767 768
        printHelp();
        errorPrint("%s", "\n\t-p need a number following!\n");
        exit(EXIT_FAILURE);
      }
769
      arguments->port = atoi(argv[++i]);
770 771 772 773 774 775 776 777
    } else if (strcmp(argv[i], "-I") == 0) {
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-I need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
      ++i;
      if (0 == strcasecmp(argv[i], "taosc")) {
S
Shuduo Sang 已提交
778
          arguments->iface = TAOSC_IFACE;
779
      } else if (0 == strcasecmp(argv[i], "rest")) {
S
Shuduo Sang 已提交
780
          arguments->iface = REST_IFACE;
781
      } else if (0 == strcasecmp(argv[i], "stmt")) {
S
Shuduo Sang 已提交
782
          arguments->iface = STMT_IFACE;
783 784 785 786
      } else {
        errorPrint("%s", "\n\t-I need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
787
    } else if (strcmp(argv[i], "-u") == 0) {
H
Haojun Liao 已提交
788 789 790 791 792
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-u need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
793 794
      arguments->user = argv[++i];
    } else if (strcmp(argv[i], "-P") == 0) {
H
Haojun Liao 已提交
795 796 797 798 799
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-P need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
800 801
      arguments->password = argv[++i];
    } else if (strcmp(argv[i], "-o") == 0) {
H
Haojun Liao 已提交
802 803 804 805 806
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-o need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
807 808
      arguments->output_file = argv[++i];
    } else if (strcmp(argv[i], "-s") == 0) {
H
Haojun Liao 已提交
809 810 811 812 813
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-s need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
814 815
      arguments->sqlFile = argv[++i];
    } else if (strcmp(argv[i], "-q") == 0) {
816 817
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
818
        printHelp();
819
        errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n");
820 821
        exit(EXIT_FAILURE);
      }
822
      arguments->async_mode = atoi(argv[++i]);
823
    } else if (strcmp(argv[i], "-T") == 0) {
824 825
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
826 827 828 829
        printHelp();
        errorPrint("%s", "\n\t-T need a number following!\n");
        exit(EXIT_FAILURE);
      }
830 831
      arguments->num_of_threads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
832 833
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
834 835 836 837
        printHelp();
        errorPrint("%s", "\n\t-i need a number following!\n");
        exit(EXIT_FAILURE);
      }
838
      arguments->insert_interval = atoi(argv[++i]);
839
    } else if (strcmp(argv[i], "-qt") == 0) {
840
      if ((argc == i+1)
841
              || (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
842
        printHelp();
843
        errorPrint("%s", "\n\t-qt need a number following!\n");
H
Haojun Liao 已提交
844 845
        exit(EXIT_FAILURE);
      }
846
      arguments->query_times = atoi(argv[++i]);
847
    } else if (strcmp(argv[i], "-B") == 0) {
848 849
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
850 851 852 853
        printHelp();
        errorPrint("%s", "\n\t-B need a number following!\n");
        exit(EXIT_FAILURE);
      }
854
      arguments->interlace_rows = atoi(argv[++i]);
855
    } else if (strcmp(argv[i], "-r") == 0) {
856 857
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
858 859 860 861
        printHelp();
        errorPrint("%s", "\n\t-r need a number following!\n");
        exit(EXIT_FAILURE);
      }
862 863
      arguments->num_of_RPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-t") == 0) {
864 865 866 867 868 869
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-t need a number following!\n");
        exit(EXIT_FAILURE);
      }
870 871
      arguments->num_of_tables = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
872 873 874 875 876 877
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-n need a number following!\n");
        exit(EXIT_FAILURE);
      }
878 879
      arguments->num_of_DPT = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-d") == 0) {
H
Haojun Liao 已提交
880 881 882 883 884
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-d need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
885 886
      arguments->database = argv[++i];
    } else if (strcmp(argv[i], "-l") == 0) {
887 888 889 890 891 892
      if (argc == i+1) {
        if (!isStringNumber(argv[i+1])) {
            printHelp();
            errorPrint("%s", "\n\t-l need a number following!\n");
            exit(EXIT_FAILURE);
        }
893
      }
894
      arguments->num_of_CPR = atoi(argv[++i]);
895 896 897 898 899 900 901 902 903 904 905

      if (arguments->num_of_CPR > MAX_NUM_DATATYPE) {
          printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE);
          prompt();
          arguments->num_of_CPR = MAX_NUM_DATATYPE;
      }

      for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) {
          arguments->datatype[col] = NULL;
      }

906
    } else if (strcmp(argv[i], "-b") == 0) {
907 908 909 910 911
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-b need valid string following!\n");
        exit(EXIT_FAILURE);
      }
912 913
      ++i;
      if (strstr(argv[i], ",") == NULL) {
914
        // only one col
915
        if (strcasecmp(argv[i], "INT")
916 917 918 919 920 921 922 923
                && strcasecmp(argv[i], "FLOAT")
                && strcasecmp(argv[i], "TINYINT")
                && strcasecmp(argv[i], "BOOL")
                && strcasecmp(argv[i], "SMALLINT")
                && strcasecmp(argv[i], "BIGINT")
                && strcasecmp(argv[i], "DOUBLE")
                && strcasecmp(argv[i], "BINARY")
                && strcasecmp(argv[i], "NCHAR")) {
924
          printHelp();
925
          errorPrint("%s", "-b: Invalid data_type!\n");
926
          exit(EXIT_FAILURE);
927
        }
928
        arguments->datatype[0] = argv[i];
929
      } else {
930
        // more than one col
931
        int index = 0;
932 933
        g_dupstr = strdup(argv[i]);
        char *running = g_dupstr;
934
        char *token = strsep(&running, ",");
935
        while(token != NULL) {
936
          if (strcasecmp(token, "INT")
937 938 939 940 941 942 943 944
                  && strcasecmp(token, "FLOAT")
                  && strcasecmp(token, "TINYINT")
                  && strcasecmp(token, "BOOL")
                  && strcasecmp(token, "SMALLINT")
                  && strcasecmp(token, "BIGINT")
                  && strcasecmp(token, "DOUBLE")
                  && strcasecmp(token, "BINARY")
                  && strcasecmp(token, "NCHAR")) {
945
            printHelp();
946
            free(g_dupstr);
947
            errorPrint("%s", "-b: Invalid data_type!\n");
948
            exit(EXIT_FAILURE);
H
hzcheng 已提交
949
          }
950
          arguments->datatype[index++] = token;
951 952
          token = strsep(&running, ",");
          if (index >= MAX_NUM_DATATYPE) break;
S
TD-1057  
Shengliang Guan 已提交
953
        }
954
        arguments->datatype[index] = NULL;
955
      }
956
    } else if (strcmp(argv[i], "-w") == 0) {
957
      if ((argc == i+1) ||
958
        (!isStringNumber(argv[i+1]))) {
959 960 961 962
        printHelp();
        errorPrint("%s", "\n\t-w need a number following!\n");
        exit(EXIT_FAILURE);
      }
963 964
      arguments->len_of_binary = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-m") == 0) {
H
Haojun Liao 已提交
965
      if ((argc == i+1) ||
966
        (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
967 968 969 970
        printHelp();
        errorPrint("%s", "\n\t-m need a number following!\n");
        exit(EXIT_FAILURE);
      }
971
      arguments->tb_prefix = argv[++i];
972 973
    } else if (strcmp(argv[i], "-N") == 0) {
      arguments->use_metric = false;
974
    } else if (strcmp(argv[i], "-x") == 0) {
975
      arguments->insert_only = false;
976
    } else if (strcmp(argv[i], "-y") == 0) {
977
      arguments->answer_yes = true;
978 979
    } else if (strcmp(argv[i], "-g") == 0) {
      arguments->debug_print = true;
980 981
    } else if (strcmp(argv[i], "-gg") == 0) {
      arguments->verbose_print = true;
982 983
    } else if (strcmp(argv[i], "-pp") == 0) {
      arguments->performance_print = true;
984
    } else if (strcmp(argv[i], "-O") == 0) {
985 986 987 988 989 990
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-O need a number following!\n");
        exit(EXIT_FAILURE);
      }
991

992
      arguments->disorderRatio = atoi(argv[++i]);
993

994
      if (arguments->disorderRatio > 50) {
995
        arguments->disorderRatio = 50;
996
      }
997

998
      if (arguments->disorderRatio < 0) {
999
        arguments->disorderRatio = 0;
1000
      }
1001

1002
    } else if (strcmp(argv[i], "-R") == 0) {
1003 1004 1005 1006 1007 1008
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-R need a number following!\n");
        exit(EXIT_FAILURE);
      }
1009

1010
      arguments->disorderRange = atoi(argv[++i]);
1011 1012 1013
      if (arguments->disorderRange < 0)
        arguments->disorderRange = 1000;

1014
    } else if (strcmp(argv[i], "-a") == 0) {
1015 1016 1017 1018 1019 1020
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-a need a number following!\n");
        exit(EXIT_FAILURE);
      }
1021 1022 1023
      arguments->replica = atoi(argv[++i]);
      if (arguments->replica > 3 || arguments->replica < 1) {
          arguments->replica = 1;
1024
      }
1025 1026
    } else if (strcmp(argv[i], "-D") == 0) {
      arguments->method_of_delete = atoi(argv[++i]);
1027 1028 1029
      if (arguments->method_of_delete > 3) {
        errorPrint("%s", "\n\t-D need a valud (0~3) number following!\n");
        exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1030
      }
1031 1032 1033 1034
    } else if ((strcmp(argv[i], "--version") == 0) ||
        (strcmp(argv[i], "-V") == 0)){
      printVersion();
      exit(0);
1035 1036 1037 1038 1039
    } else if (strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else {
      printHelp();
1040
      errorPrint("%s", "ERROR: wrong options\n");
1041
      exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1042 1043
    }
  }
1044

1045
  if (((arguments->debug_print) && (arguments->metaFile == NULL))
1046
          || arguments->verbose_print) {
1047
    printf("###################################################################\n");
1048
    printf("# meta file:                         %s\n", arguments->metaFile);
1049
    printf("# Server IP:                         %s:%hu\n",
1050 1051
            arguments->host == NULL ? "localhost" : arguments->host,
            arguments->port );
1052 1053
    printf("# User:                              %s\n", arguments->user);
    printf("# Password:                          %s\n", arguments->password);
1054 1055
    printf("# Use metric:                        %s\n",
            arguments->use_metric ? "true" : "false");
1056
    if (*(arguments->datatype)) {
1057 1058 1059 1060 1061 1062 1063
        printf("# Specified data type:               ");
        for (int i = 0; i < MAX_NUM_DATATYPE; i++)
            if (arguments->datatype[i])
               printf("%s,", arguments->datatype[i]);
            else
                break;
        printf("\n");
1064
    }
1065
    printf("# Insertion interval:                %"PRIu64"\n",
1066
            arguments->insert_interval);
1067
    printf("# Number of records per req:         %"PRIu64"\n",
1068
            arguments->num_of_RPR);
1069
    printf("# Max SQL length:                    %"PRIu64"\n",
1070
            arguments->max_sql_len);
1071
    printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
1072
    printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
1073
    printf("# Number of Tables:                  %"PRId64"\n",
1074
            arguments->num_of_tables);
1075
    printf("# Number of Data per Table:          %"PRId64"\n",
1076
            arguments->num_of_DPT);
1077 1078 1079 1080 1081
    printf("# Database name:                     %s\n", arguments->database);
    printf("# Table prefix:                      %s\n", arguments->tb_prefix);
    if (arguments->disorderRatio) {
      printf("# Data order:                        %d\n", arguments->disorderRatio);
      printf("# Data out of order rate:            %d\n", arguments->disorderRange);
1082

1083 1084 1085 1086
    }
    printf("# Delete method:                     %d\n", arguments->method_of_delete);
    printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
    printf("# Print debug info:                  %d\n", arguments->debug_print);
1087
    printf("# Print verbose info:                %d\n", arguments->verbose_print);
1088
    printf("###################################################################\n");
1089 1090 1091 1092
    if (!arguments->answer_yes) {
        printf("Press enter key to continue\n\n");
        (void) getchar();
    }
1093
  }
1094
}
H
hzcheng 已提交
1095

1096 1097
static bool getInfoFromJsonFile(char* file);
static void init_rand_data();
1098
static void tmfclose(FILE *fp) {
1099 1100 1101 1102
  if (NULL != fp) {
    fclose(fp);
  }
}
1103

1104
static void tmfree(char *buf) {
1105 1106
  if (NULL != buf) {
    free(buf);
H
Hui Li 已提交
1107
  }
1108
}
H
Hui Li 已提交
1109

1110
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
1111 1112 1113
  int i;
  TAOS_RES *res = NULL;
  int32_t   code = -1;
1114

S
Shuduo Sang 已提交
1115
  for (i = 0; i < 5 /* retry */; i++) {
1116 1117 1118
    if (NULL != res) {
      taos_free_result(res);
      res = NULL;
H
hzcheng 已提交
1119
    }
1120

1121 1122 1123 1124
    res = taos_query(taos, command);
    code = taos_errno(res);
    if (0 == code) {
      break;
1125
    }
H
hzcheng 已提交
1126 1127
  }

1128
  if (code != 0) {
1129 1130
    if (!quiet) {
      debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
1131
      errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res));
1132
    }
1133 1134 1135
    taos_free_result(res);
    //taos_close(taos);
    return -1;
1136
  }
H
hzcheng 已提交
1137

1138 1139 1140 1141
  if (INSERT_TYPE == type) {
    int affectedRows = taos_affected_rows(res);
    taos_free_result(res);
    return affectedRows;
H
hzcheng 已提交
1142
  }
1143

1144
  taos_free_result(res);
1145 1146
  return 0;
}
H
hzcheng 已提交
1147

1148 1149
static void appendResultBufToFile(char *resultBuf, char *resultFile)
{
1150
  FILE *fp = NULL;
1151 1152
  if (resultFile[0] != 0) {
    fp = fopen(resultFile, "at");
1153
    if (fp == NULL) {
1154 1155 1156 1157
      errorPrint(
              "%s() LN%d, failed to open result file: %s, result will not save to file\n",
              __func__, __LINE__, resultFile);
      return;
1158 1159
    }
  }
1160

1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
  fprintf(fp, "%s", resultBuf);
  tmfclose(fp);
}

static void appendResultToFile(TAOS_RES *res, char* resultFile) {
  TAOS_ROW    row = NULL;
  int         num_rows = 0;
  int         num_fields = taos_field_count(res);
  TAOS_FIELD *fields     = taos_fetch_fields(res);

1171 1172
  char* databuf = (char*) calloc(1, 100*1024*1024);
  if (databuf == NULL) {
1173 1174
    errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n",
            __func__, __LINE__);
1175 1176
    return ;
  }
H
hzcheng 已提交
1177

1178 1179
  int   totalLen = 0;
  char  temp[16000];
H
hzcheng 已提交
1180

1181
  // fetch the records row by row
1182
  while((row = taos_fetch_row(res))) {
1183
    if (totalLen >= 100*1024*1024 - 32000) {
1184
      appendResultBufToFile(databuf, resultFile);
1185 1186
      totalLen = 0;
      memset(databuf, 0, 100*1024*1024);
H
hzcheng 已提交
1187
    }
1188 1189 1190 1191 1192 1193
    num_rows++;
    int len = taos_print_row(temp, row, fields, num_fields);
    len += sprintf(temp + len, "\n");
    //printf("query result:%s\n", temp);
    memcpy(databuf + totalLen, temp, len);
    totalLen += len;
H
hzcheng 已提交
1194 1195
  }

1196 1197
  verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n",
          __func__, __LINE__, databuf, resultFile);
1198
  appendResultBufToFile(databuf, resultFile);
1199 1200
  free(databuf);
}
H
hzcheng 已提交
1201

1202
static void selectAndGetResult(
1203 1204
        threadInfo *pThreadInfo, char *command)
{
1205 1206 1207 1208 1209 1210 1211 1212 1213
  if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) {
    TAOS_RES *res = taos_query(pThreadInfo->taos, command);
    if (res == NULL || taos_errno(res) != 0) {
        errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n",
            __func__, __LINE__, command, taos_errstr(res));
        taos_free_result(res);
        return;
    }

1214 1215
    if ((strlen(pThreadInfo->fp))) {
      appendResultToFile(res, pThreadInfo->fp);
1216
    }
1217
    taos_free_result(res);
1218

1219 1220 1221
  } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
      int retCode = postProceSql(
              g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port,
1222
              command,
1223
              pThreadInfo->fp);
1224 1225 1226 1227 1228 1229 1230 1231
      if (0 != retCode) {
        printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID);
      }

  } else {
      errorPrint("%s() LN%d, unknown query mode: %s\n",
        __func__, __LINE__, g_queryInfo.queryMode);
  }
1232
}
H
hzcheng 已提交
1233

1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272
static int32_t rand_bool(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 2;
}

static int32_t rand_tinyint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 128;
}

static int32_t rand_smallint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 32767;
}

static int32_t rand_int(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor];
}

static int64_t rand_bigint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randbigint[cursor];
}

static float rand_float(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
1273
  return randfloat[cursor];
1274 1275
}

1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
#if 0
static const char charNum[] = "0123456789";

static void nonrand_string(char *, int) __attribute__ ((unused));   // reserve for debugging purpose
static void nonrand_string(char *str, int size)
{
  str[0] = 0;
  if (size > 0) {
    int n;
    for (n = 0; n < size; n++) {
      str[n] = charNum[n % 10];
    }
    str[n] = 0;
  }
}
#endif

1293
static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1294

1295
static void rand_string(char *str, int size) {
1296 1297 1298 1299
  str[0] = 0;
  if (size > 0) {
    //--size;
    int n;
1300
    for (n = 0; n < size; n++) {
1301
      int key = abs(rand_tinyint()) % (int)(sizeof(charset) - 1);
1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317
      str[n] = charset[key];
    }
    str[n] = 0;
  }
}

static double rand_double() {
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randdouble[cursor];

}

static void init_rand_data() {
  for (int i = 0; i < MAX_PREPARED_RAND; i++){
1318 1319 1320 1321
    randint[i] = (int)(taosRandom() % 65535);
    randbigint[i] = (int64_t)(taosRandom() % 2147483648);
    randfloat[i] = (float)(taosRandom() / 1000.0);
    randdouble[i] = (double)(taosRandom() / 1000000.0);
1322 1323 1324
  }
}

1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1345
static int printfInsertMeta() {
1346
    SHOW_PARSE_RESULT_START();
1347

1348
  printf("interface:                  \033[33m%s\033[0m\n",
S
Shuduo Sang 已提交
1349
          (g_args.iface==TAOSC_IFACE)?"taosc":(g_args.iface==REST_IFACE)?"rest":"stmt");
1350 1351 1352
  printf("host:                       \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port);
  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
1353
  printf("configDir:                  \033[33m%s\033[0m\n", configDir);
1354 1355 1356
  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
  printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl);
1357
  printf("top insert interval:        \033[33m%"PRIu64"\033[0m\n",
1358
          g_args.insert_interval);
1359
  printf("number of records per req:  \033[33m%"PRIu64"\033[0m\n",
1360
          g_args.num_of_RPR);
1361
  printf("max sql length:             \033[33m%"PRIu64"\033[0m\n",
1362
          g_args.max_sql_len);
1363 1364

  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
1365

1366 1367
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    printf("database[\033[33m%d\033[0m]:\n", i);
1368
    printf("  database[%d] name:      \033[33m%s\033[0m\n", i, g_Dbs.db[i].dbName);
1369
    if (0 == g_Dbs.db[i].drop) {
1370 1371 1372
      printf("  drop:                  \033[33mno\033[0m\n");
    } else {
      printf("  drop:                  \033[33myes\033[0m\n");
1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      printf("  blocks:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      printf("  cache:                 \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      printf("  days:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      printf("  keep:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      printf("  replica:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      printf("  update:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      printf("  minRows:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      printf("  maxRows:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      printf("  walLevel:              \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      printf("  fsync:                 \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      printf("  quorum:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1412
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
1413
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1414
        printf("  precision:             \033[33m%s\033[0m\n",
1415
            g_Dbs.db[i].dbCfg.precision);
1416
      } else {
1417
        printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
1418
                g_Dbs.db[i].dbCfg.precision);
1419
        return -1;
1420 1421 1422
      }
    }

1423
    printf("  super table count:     \033[33m%"PRIu64"\033[0m\n",
1424
        g_Dbs.db[i].superTblCount);
1425 1426
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      printf("  super table[\033[33m%"PRIu64"\033[0m]:\n", j);
1427

1428
      printf("      stbName:           \033[33m%s\033[0m\n",
1429
          g_Dbs.db[i].superTbls[j].sTblName);
1430 1431 1432 1433 1434 1435 1436 1437

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
      }
1438

1439 1440 1441 1442 1443 1444 1445
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
      }
1446

1447
      printf("      childTblCount:     \033[33m%"PRId64"\033[0m\n",
1448
              g_Dbs.db[i].superTbls[j].childTblCount);
1449
      printf("      childTblPrefix:    \033[33m%s\033[0m\n",
1450
              g_Dbs.db[i].superTbls[j].childTblPrefix);
1451
      printf("      dataSource:        \033[33m%s\033[0m\n",
1452
              g_Dbs.db[i].superTbls[j].dataSource);
1453
      printf("      insertMode:        \033[33m%s\033[0m\n",
1454 1455
          (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc":
          (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt");
1456
      if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
1457
        printf("      childTblLimit:     \033[33m%"PRId64"\033[0m\n",
1458
                g_Dbs.db[i].superTbls[j].childTblLimit);
1459
      }
1460
      if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) {
1461
        printf("      childTblOffset:    \033[33m%"PRIu64"\033[0m\n",
1462
                g_Dbs.db[i].superTbls[j].childTblOffset);
1463
      }
1464
      printf("      insertRows:        \033[33m%"PRId64"\033[0m\n",
1465
              g_Dbs.db[i].superTbls[j].insertRows);
1466
/*
1467
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1468
        printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
1469
      }else {
1470
        printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
1471
      }
1472
      */
1473
      printf("      interlaceRows:     \033[33m%"PRIu64"\033[0m\n",
1474
              g_Dbs.db[i].superTbls[j].interlaceRows);
1475 1476

      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1477
        printf("      stable insert interval:   \033[33m%"PRIu64"\033[0m\n",
1478
            g_Dbs.db[i].superTbls[j].insertInterval);
1479 1480
      }

1481
      printf("      disorderRange:     \033[33m%d\033[0m\n",
1482
              g_Dbs.db[i].superTbls[j].disorderRange);
1483
      printf("      disorderRatio:     \033[33m%d\033[0m\n",
1484
              g_Dbs.db[i].superTbls[j].disorderRatio);
1485
      printf("      maxSqlLen:         \033[33m%"PRIu64"\033[0m\n",
1486
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1487
      printf("      timeStampStep:     \033[33m%"PRId64"\033[0m\n",
1488
              g_Dbs.db[i].superTbls[j].timeStampStep);
1489
      printf("      startTimestamp:    \033[33m%s\033[0m\n",
1490
              g_Dbs.db[i].superTbls[j].startTimestamp);
1491
      printf("      sampleFormat:      \033[33m%s\033[0m\n",
1492
              g_Dbs.db[i].superTbls[j].sampleFormat);
1493
      printf("      sampleFile:        \033[33m%s\033[0m\n",
1494
              g_Dbs.db[i].superTbls[j].sampleFile);
1495
      printf("      tagsFile:          \033[33m%s\033[0m\n",
1496
              g_Dbs.db[i].superTbls[j].tagsFile);
1497
      printf("      columnCount:       \033[33m%d\033[0m\n",
1498
              g_Dbs.db[i].superTbls[j].columnCount);
1499 1500
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1501
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
1502 1503 1504
                       "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "nchar", 5))) {
1505
          printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
1506 1507
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1508
        } else {
1509
          printf("column[%d]:\033[33m%s\033[0m ", k,
1510
                  g_Dbs.db[i].superTbls[j].columns[k].dataType);
1511 1512 1513
        }
      }
      printf("\n");
1514 1515

      printf("      tagCount:            \033[33m%d\033[0m\n        ",
1516
              g_Dbs.db[i].superTbls[j].tagCount);
1517 1518
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1519
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
1520 1521 1522
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1523
          printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
1524 1525
                  g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1526
        } else {
1527
          printf("tag[%d]:\033[33m%s\033[0m ", k,
1528
                  g_Dbs.db[i].superTbls[j].tags[k].dataType);
1529
        }
1530 1531 1532 1533 1534
      }
      printf("\n");
    }
    printf("\n");
  }
1535 1536

  SHOW_PARSE_RESULT_END();
1537 1538

  return 0;
1539 1540 1541
}

static void printfInsertMetaToFile(FILE* fp) {
1542 1543

  SHOW_PARSE_RESULT_START_TO_FILE(fp);
1544

1545 1546
  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
  fprintf(fp, "user:                       %s\n", g_Dbs.user);
1547
  fprintf(fp, "configDir:                  %s\n", configDir);
1548 1549 1550
  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
1551 1552
  fprintf(fp, "number of records per req:  %"PRIu64"\n", g_args.num_of_RPR);
  fprintf(fp, "max sql length:             %"PRIu64"\n", g_args.max_sql_len);
1553
  fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
1554

1555 1556
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    fprintf(fp, "database[%d]:\n", i);
1557
    fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
1558
    if (0 == g_Dbs.db[i].drop) {
1559
      fprintf(fp, "  drop:                  no\n");
1560
    }else {
1561
      fprintf(fp, "  drop:                  yes\n");
1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1601
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
1602
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1603 1604 1605 1606 1607 1608
        fprintf(fp, "  precision:             %s\n", g_Dbs.db[i].dbCfg.precision);
      } else {
        fprintf(fp, "  precision error:       %s\n", g_Dbs.db[i].dbCfg.precision);
      }
    }

1609
    fprintf(fp, "  super table count:     %"PRIu64"\n", g_Dbs.db[i].superTblCount);
S
Shuduo Sang 已提交
1610 1611
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      fprintf(fp, "  super table[%"PRIu64"]:\n", j);
1612 1613

      fprintf(fp, "      stbName:           %s\n",  g_Dbs.db[i].superTbls[j].sTblName);
1614 1615 1616 1617 1618 1619 1620 1621

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
      } else {
        fprintf(fp, "      autoCreateTable:   %s\n",  "error");
      }
1622

1623 1624 1625 1626 1627 1628 1629
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "yes");
      } else {
        fprintf(fp, "      childTblExists:    %s\n",  "error");
      }
1630

1631
      fprintf(fp, "      childTblCount:     %"PRId64"\n",
1632 1633 1634 1635 1636 1637
              g_Dbs.db[i].superTbls[j].childTblCount);
      fprintf(fp, "      childTblPrefix:    %s\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      fprintf(fp, "      dataSource:        %s\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      fprintf(fp, "      insertMode:        %s\n",
1638 1639
          (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc":
          (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt");
1640
      fprintf(fp, "      insertRows:        %"PRId64"\n",
1641
              g_Dbs.db[i].superTbls[j].insertRows);
1642
      fprintf(fp, "      interlace rows:    %"PRIu64"\n",
1643
              g_Dbs.db[i].superTbls[j].interlaceRows);
1644
      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1645
        fprintf(fp, "      stable insert interval:   %"PRIu64"\n",
1646
                g_Dbs.db[i].superTbls[j].insertInterval);
1647
      }
1648
/*
1649
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1650
        fprintf(fp, "      multiThreadWriteOneTbl:  no\n");
1651
      }else {
1652
        fprintf(fp, "      multiThreadWriteOneTbl:  yes\n");
1653
      }
1654
      */
1655
      fprintf(fp, "      interlaceRows:     %"PRIu64"\n",
1656
              g_Dbs.db[i].superTbls[j].interlaceRows);
1657 1658 1659 1660
      fprintf(fp, "      disorderRange:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      fprintf(fp, "      disorderRatio:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
1661
      fprintf(fp, "      maxSqlLen:         %"PRIu64"\n",
1662
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1663

1664 1665 1666 1667
      fprintf(fp, "      timeStampStep:     %"PRId64"\n",
              g_Dbs.db[i].superTbls[j].timeStampStep);
      fprintf(fp, "      startTimestamp:    %s\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
1668 1669 1670 1671 1672 1673
      fprintf(fp, "      sampleFormat:      %s\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      fprintf(fp, "      sampleFile:        %s\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      fprintf(fp, "      tagsFile:          %s\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
1674

1675 1676
      fprintf(fp, "      columnCount:       %d\n        ",
              g_Dbs.db[i].superTbls[j].columnCount);
1677 1678
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1679
        if ((0 == strncasecmp(
1680 1681 1682 1683
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "nchar", strlen("nchar")))) {
1684 1685 1686
          fprintf(fp, "column[%d]:%s(%d) ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1687
        } else {
1688 1689
          fprintf(fp, "column[%d]:%s ",
                  k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
1690 1691 1692
        }
      }
      fprintf(fp, "\n");
1693 1694 1695

      fprintf(fp, "      tagCount:            %d\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1696 1697
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1698
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
1699 1700 1701
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1702 1703
          fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1704 1705
        } else {
          fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
1706
        }
1707 1708 1709 1710 1711
      }
      fprintf(fp, "\n");
    }
    fprintf(fp, "\n");
  }
1712

1713
  SHOW_PARSE_RESULT_END_TO_FILE(fp);
1714 1715 1716
}

static void printfQueryMeta() {
1717

1718
  SHOW_PARSE_RESULT_START();
1719

1720
  printf("host:                    \033[33m%s:%u\033[0m\n",
1721
          g_queryInfo.host, g_queryInfo.port);
1722 1723 1724 1725
  printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
  printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

  printf("\n");
1726 1727 1728 1729

  if ((SUBSCRIBE_TEST == g_args.test_mode) || (QUERY_TEST == g_args.test_mode)) {
    printf("specified table query info:                   \n");
    printf("sqlCount:       \033[33m%"PRIu64"\033[0m\n",
1730
      g_queryInfo.specifiedQueryInfo.sqlCount);
1731 1732 1733
    if (g_queryInfo.specifiedQueryInfo.sqlCount > 0) {
      printf("specified tbl query times:\n");
      printf("                \033[33m%"PRIu64"\033[0m\n",
1734
      g_queryInfo.specifiedQueryInfo.queryTimes);
1735 1736 1737 1738 1739 1740
      printf("query interval: \033[33m%"PRIu64" ms\033[0m\n",
        g_queryInfo.specifiedQueryInfo.queryInterval);
      printf("top query times:\033[33m%"PRIu64"\033[0m\n", g_args.query_times);
      printf("concurrent:     \033[33m%"PRIu64"\033[0m\n",
      g_queryInfo.specifiedQueryInfo.concurrent);
      printf("mod:            \033[33m%s\033[0m\n",
1741
        (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync");
1742
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1743
        g_queryInfo.specifiedQueryInfo.subscribeInterval);
1744
      printf("restart:        \033[33m%d\033[0m\n",
1745
        g_queryInfo.specifiedQueryInfo.subscribeRestart);
1746
      printf("keepProgress:   \033[33m%d\033[0m\n",
1747
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
1748

1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764
      for (uint64_t i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        printf("  sql[%"PRIu64"]: \033[33m%s\033[0m\n",
            i, g_queryInfo.specifiedQueryInfo.sql[i]);
      }
      printf("\n");
    }

    printf("super table query info:\n");
    printf("sqlCount:       \033[33m%"PRIu64"\033[0m\n",
      g_queryInfo.superQueryInfo.sqlCount);

    if (g_queryInfo.superQueryInfo.sqlCount > 0) {
      printf("query interval: \033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryInterval);
      printf("threadCnt:      \033[33m%d\033[0m\n",
        g_queryInfo.superQueryInfo.threadCnt);
1765
      printf("childTblCount:  \033[33m%"PRId64"\033[0m\n",
1766 1767 1768 1769 1770 1771 1772
        g_queryInfo.superQueryInfo.childTblCount);
      printf("stable name:    \033[33m%s\033[0m\n",
        g_queryInfo.superQueryInfo.sTblName);
      printf("stb query times:\033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryTimes);

      printf("mod:            \033[33m%s\033[0m\n",
1773
        (g_queryInfo.superQueryInfo.asyncMode)?"async":"sync");
1774
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1775
        g_queryInfo.superQueryInfo.subscribeInterval);
1776
      printf("restart:        \033[33m%d\033[0m\n",
1777
        g_queryInfo.superQueryInfo.subscribeRestart);
1778
      printf("keepProgress:   \033[33m%d\033[0m\n",
1779
        g_queryInfo.superQueryInfo.subscribeKeepProgress);
1780

1781 1782 1783 1784 1785 1786
      for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
        printf("  sql[%d]: \033[33m%s\033[0m\n",
            i, g_queryInfo.superQueryInfo.sql[i]);
      }
      printf("\n");
    }
1787
  }
1788

1789
  SHOW_PARSE_RESULT_END();
1790 1791
}

1792
static char* formatTimestamp(char* buf, int64_t val, int precision) {
1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823
  time_t tt;
  if (precision == TSDB_TIME_PRECISION_MICRO) {
    tt = (time_t)(val / 1000000);
  } else {
    tt = (time_t)(val / 1000);
  }

/* comment out as it make testcases like select_with_tags.sim fail.
  but in windows, this may cause the call to localtime crash if tt < 0,
  need to find a better solution.
  if (tt < 0) {
    tt = 0;
  }
  */

#ifdef WINDOWS
  if (tt < 0) tt = 0;
#endif

  struct tm* ptm = localtime(&tt);
  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);

  if (precision == TSDB_TIME_PRECISION_MICRO) {
    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
  } else {
    sprintf(buf + pos, ".%03d", (int)(val % 1000));
  }

  return buf;
}

1824
static void xDumpFieldToFile(FILE* fp, const char* val,
1825
        TAOS_FIELD* field, int32_t length, int precision) {
1826

1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861
  if (val == NULL) {
    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
    return;
  }

  char buf[TSDB_MAX_BYTES_PER_ROW];
  switch (field->type) {
    case TSDB_DATA_TYPE_BOOL:
      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
      break;
    case TSDB_DATA_TYPE_TINYINT:
      fprintf(fp, "%d", *((int8_t *)val));
      break;
    case TSDB_DATA_TYPE_SMALLINT:
      fprintf(fp, "%d", *((int16_t *)val));
      break;
    case TSDB_DATA_TYPE_INT:
      fprintf(fp, "%d", *((int32_t *)val));
      break;
    case TSDB_DATA_TYPE_BIGINT:
      fprintf(fp, "%" PRId64, *((int64_t *)val));
      break;
    case TSDB_DATA_TYPE_FLOAT:
      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
      break;
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      memcpy(buf, val, length);
      buf[length] = 0;
      fprintf(fp, "\'%s\'", buf);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
1862
      formatTimestamp(buf, *(int64_t*)val, precision);
1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877
      fprintf(fp, "'%s'", buf);
      break;
    default:
      break;
  }
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  FILE* fp = fopen(fname, "at");
  if (fp == NULL) {
1878
    errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892
    return -1;
  }

  int num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int precision = taos_result_precision(tres);

  for (int col = 0; col < num_fields; col++) {
    if (col > 0) {
      fprintf(fp, ",");
    }
    fprintf(fp, "%s", fields[col].name);
  }
  fputc('\n', fp);
1893

1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913
  int numOfRows = 0;
  do {
    int32_t* length = taos_fetch_lengths(tres);
    for (int i = 0; i < num_fields; i++) {
      if (i > 0) {
        fputc(',', fp);
      }
      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
    }
    fputc('\n', fp);

    numOfRows++;
    row = taos_fetch_row(tres);
  } while( row != NULL);

  fclose(fp);

  return numOfRows;
}

1914 1915
static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
  TAOS_RES * res;
1916 1917
  TAOS_ROW row = NULL;
  int count = 0;
1918 1919

  res = taos_query(taos, "show databases;");
1920
  int32_t code = taos_errno(res);
1921

1922
  if (code != 0) {
1923
    errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
1924 1925 1926 1927 1928
    return -1;
  }

  TAOS_FIELD *fields = taos_fetch_fields(res);

1929
  while((row = taos_fetch_row(res)) != NULL) {
1930
    // sys database name : 'log'
1931
    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log",
1932
                fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) {
1933 1934
      continue;
    }
1935 1936 1937

    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
    if (dbInfos[count] == NULL) {
1938
      errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
1939 1940 1941
      return -1;
    }

1942
    tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
1943
            fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
1944
    formatTimestamp(dbInfos[count]->create_time,
1945 1946
            *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
            TSDB_TIME_PRECISION_MILLI);
S
Shuduo Sang 已提交
1947
    dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
1948
    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
1949 1950
    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
1951
    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
1952

1953
    tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
1954
            fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
1955 1956 1957 1958 1959 1960 1961
    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
1962
    dbInfos[count]->cachelast =
1963
      (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));
1964

1965
    tstrncpy(dbInfos[count]->precision,
1966 1967
            (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
            fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
1968
    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
1969
    tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
1970
            fields[TSDB_SHOW_DB_STATUS_INDEX].bytes);
1971

1972 1973
    count++;
    if (count > MAX_DATABASE_COUNT) {
1974
      errorPrint("%s() LN%d, The database count overflow than %d\n",
1975
         __func__, __LINE__, MAX_DATABASE_COUNT);
1976 1977 1978 1979 1980 1981 1982
      break;
    }
  }

  return count;
}

1983
static void printfDbInfoForQueryToFile(
1984
        char* filename, SDbInfo* dbInfos, int index) {
1985

1986
  if (filename[0] == 0)
1987
      return;
1988 1989 1990

  FILE *fp = fopen(filename, "at");
  if (fp == NULL) {
1991
    errorPrint( "failed to open file: %s\n", filename);
1992
    return;
1993 1994 1995 1996 1997
  }

  fprintf(fp, "================ database[%d] ================\n", index);
  fprintf(fp, "name: %s\n", dbInfos->name);
  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
S
Shuduo Sang 已提交
1998
  fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables);
1999
  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);
2000 2001
  fprintf(fp, "replica: %d\n", dbInfos->replica);
  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
2002 2003
  fprintf(fp, "days: %d\n", dbInfos->days);
  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);
2004 2005 2006 2007 2008 2009 2010
  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
  fprintf(fp, "comp: %d\n", dbInfos->comp);
2011 2012
  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);
  fprintf(fp, "precision: %s\n", dbInfos->precision);
2013
  fprintf(fp, "update: %d\n", dbInfos->update);
2014
  fprintf(fp, "status: %s\n", dbInfos->status);
2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028
  fprintf(fp, "\n");

  fclose(fp);
}

static void printfQuerySystemInfo(TAOS * taos) {
  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
  TAOS_RES* res;

  time_t t;
  struct tm* lt;
  time(&t);
  lt = localtime(&t);
2029
  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
2030 2031
          lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
          lt->tm_sec);
2032

2033 2034
  // show variables
  res = taos_query(taos, "show variables;");
2035
  //appendResultToFile(res, filename);
2036 2037 2038 2039 2040
  xDumpResultToFile(filename, res);

  // show dnodes
  res = taos_query(taos, "show dnodes;");
  xDumpResultToFile(filename, res);
2041
  //appendResultToFile(res, filename);
2042

2043 2044 2045 2046
  // show databases
  res = taos_query(taos, "show databases;");
  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
  if (dbInfos == NULL) {
2047
    errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
2048 2049 2050
    return;
  }
  int dbCount = getDbFromServer(taos, dbInfos);
2051
  if (dbCount <= 0) {
2052 2053
      free(dbInfos);
      return;
2054
  }
2055 2056

  for (int i = 0; i < dbCount; i++) {
2057
    // printf database info
2058
    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
2059

2060 2061 2062 2063
    // show db.vgroups
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);
2064

2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075
    // show db.stables
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);

    free(dbInfos[i]);
  }

  free(dbInfos);
}

2076 2077
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
        char* sqlstr, char *resultFile)
2078
{
2079
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
2080

2081
    char *url = "/rest/sql";
2082

2083 2084 2085 2086
    int bytes, sent, received, req_str_len, resp_len;
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
2087

2088
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;
2089

2090 2091 2092 2093 2094
    request_buf = malloc(req_buf_len);
    if (NULL == request_buf) {
      errorPrint("%s", "ERROR, cannot allocate memory.\n");
      exit(EXIT_FAILURE);
    }
2095

2096 2097
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};
2098

2099 2100 2101 2102 2103 2104 2105 2106
    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
2107

2108 2109 2110 2111
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);
2112

2113
    char base64_buf[INPUT_BUF_LEN];
2114
#ifdef WINDOWS
2115
    WSADATA wsaData;
2116
    WSAStartup(MAKEWORD(2, 2), &wsaData);
2117 2118
    SOCKET sockfd;
#else
2119
    int sockfd;
2120
#endif
2121 2122
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (sockfd < 0) {
2123
#ifdef WINDOWS
2124
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
2125
#endif
2126 2127 2128 2129
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
        free(request_buf);
        ERROR_EXIT("ERROR opening socket");
    }
2130

2131
    int retConn = connect(sockfd, (struct sockaddr *)pServAddr, sizeof(struct sockaddr));
2132 2133 2134 2135 2136
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
        free(request_buf);
        ERROR_EXIT("ERROR connecting");
    }
2137

2138
    memset(base64_buf, 0, INPUT_BUF_LEN);
2139

2140 2141 2142 2143 2144 2145 2146 2147
    for (int n = 0, m = 0; n < userpass_buf_len;) {
      uint32_t oct_a = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t oct_b = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t oct_c = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;
2148

2149 2150 2151 2152 2153
      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }
2154

2155 2156
    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';
2157

2158 2159 2160
    debugPrint("%s() LN%d: auth string base64 encoded: %s\n",
            __func__, __LINE__, base64_buf);
    char *auth = base64_buf;
2161

2162 2163 2164 2165 2166 2167 2168 2169 2170
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
2171

2172 2173 2174
    req_str_len = strlen(request_buf);
    sent = 0;
    do {
2175
#ifdef WINDOWS
2176
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
2177
#else
2178
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
2179
#endif
2180 2181 2182 2183 2184 2185
        if (bytes < 0)
            ERROR_EXIT("ERROR writing message to socket");
        if (bytes == 0)
            break;
        sent+=bytes;
    } while(sent < req_str_len);
2186

2187 2188 2189 2190
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
    received = 0;
    do {
2191
#ifdef WINDOWS
2192
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
2193
#else
2194
        bytes = read(sockfd, response_buf + received, resp_len - received);
2195
#endif
2196 2197 2198 2199 2200 2201 2202 2203
        if (bytes < 0) {
            free(request_buf);
            ERROR_EXIT("ERROR reading response from socket");
        }
        if (bytes == 0)
            break;
        received += bytes;
    } while(received < resp_len);
2204

2205 2206 2207 2208
    if (received == resp_len) {
        free(request_buf);
        ERROR_EXIT("ERROR storing complete response from socket");
    }
H
Haojun Liao 已提交
2209

2210 2211
    response_buf[RESP_BUF_LEN - 1] = '\0';
    printf("Response:\n%s\n", response_buf);
H
Haojun Liao 已提交
2212

2213 2214 2215 2216
    if (resultFile) {
       appendResultBufToFile(response_buf, resultFile);
    }

2217
    free(request_buf);
2218
#ifdef WINDOWS
2219
    closesocket(sockfd);
2220 2221
    WSACleanup();
#else
2222
    close(sockfd);
2223
#endif
2224

2225
    return 0;
2226 2227
}

2228
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
2229 2230
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
2231
    errorPrint("%s() LN%d, calloc failed! size:%d\n",
2232
        __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
2233 2234
    return NULL;
  }
2235

2236
  int    dataLen = 0;
2237
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2238
          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);
2239

2240 2241 2242
  return dataBuf;
}

2243
static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) {
2244 2245 2246 2247 2248 2249 2250 2251 2252
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
    printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
    return NULL;
  }

  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
  for (int i = 0; i < stbInfo->tagCount; i++) {
2253
    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
2254
            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
2255
      if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
2256
        printf("binary or nchar length overflow, max size:%u\n",
2257
                (uint32_t)TSDB_MAX_BINARY_LEN);
2258 2259 2260
        tmfree(dataBuf);
        return NULL;
      }
2261

2262 2263
      int tagBufLen = stbInfo->tags[i].dataLen + 1;
      char* buf = (char*)calloc(tagBufLen, 1);
2264 2265 2266 2267 2268
      if (NULL == buf) {
        printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
        tmfree(dataBuf);
        return NULL;
      }
2269 2270 2271 2272 2273 2274 2275

      if (tableSeq % 2) {
        tstrncpy(buf, "beijing", tagBufLen);
      } else {
        tstrncpy(buf, "shanghai", tagBufLen);
      }
      //rand_string(buf, stbInfo->tags[i].dataLen);
2276
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2277
              "\'%s\', ", buf);
2278
      tmfree(buf);
2279
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2280
                "int", strlen("int"))) {
2281
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2282
              "%d, ", tableSeq);
2283
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2284
                "bigint", strlen("bigint"))) {
2285
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2286
              "%"PRId64", ", rand_bigint());
2287
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2288
                "float", strlen("float"))) {
2289
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2290
              "%f, ", rand_float());
2291
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2292
                "double", strlen("double"))) {
2293
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2294
              "%f, ", rand_double());
2295
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2296
                "smallint", strlen("smallint"))) {
2297
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2298
              "%d, ", rand_smallint());
2299
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2300
                "tinyint", strlen("tinyint"))) {
2301
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2302
              "%d, ", rand_tinyint());
2303
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2304
                "bool", strlen("bool"))) {
2305
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2306
              "%d, ", rand_bool());
2307
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2308
                "timestamp", strlen("timestamp"))) {
2309
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2310
              "%"PRId64", ", rand_bigint());
2311 2312 2313 2314 2315 2316
    }  else {
      printf("No support data type: %s\n", stbInfo->tags[i].dataType);
      tmfree(dataBuf);
      return NULL;
    }
  }
2317

2318
  dataLen -= 2;
2319
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");
2320 2321 2322
  return dataBuf;
}

2323
static int calcRowLen(SSuperTable*  superTbls) {
2324 2325
  int colIndex;
  int  lenOfOneRow = 0;
2326

2327 2328
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2329

2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
2346
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2347
      lenOfOneRow += 42;
2348
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
2362

2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
2379
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2380 2381 2382 2383 2384 2385 2386 2387
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2388

2389 2390 2391 2392
  return 0;
}


2393
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
2394
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
2395
        int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) {
2396

2397
  char command[BUFFER_SIZE] = "\0";
2398 2399
  char limitBuf[100] = "\0";

2400
  TAOS_RES * res;
2401 2402 2403
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2404 2405

  if (offset >= 0) {
2406
    snprintf(limitBuf, 100, " limit %"PRId64" offset %"PRIu64"",
2407
            limit, offset);
2408 2409
  }

2410
  //get all child table name use cmd: select tbname from superTblName;
2411
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s",
2412
          dbName, sTblName, limitBuf);
2413

2414
  res = taos_query(taos, command);
2415 2416 2417 2418
  int32_t code = taos_errno(res);
  if (code != 0) {
    taos_free_result(res);
    taos_close(taos);
2419
    errorPrint("%s() LN%d, failed to run command %s\n",
2420
           __func__, __LINE__, command);
2421 2422 2423
    exit(-1);
  }

2424 2425
  int64_t childTblCount = (limit < 0)?10000:limit;
  int64_t count = 0;
2426 2427 2428
  if (childTblName == NULL) {
    childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (NULL ==  childTblName) {
2429 2430 2431 2432
    taos_free_result(res);
        taos_close(taos);
        errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__);
        exit(-1);
2433 2434 2435
    }
  }

2436
  char* pTblName = childTblName;
2437
  while((row = taos_fetch_row(res)) != NULL) {
2438
    int32_t* len = taos_fetch_lengths(res);
2439
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2440 2441 2442
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2443
      char *tmp = realloc(childTblName,
2444
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2445 2446 2447
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2448
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
2449
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2450 2451
      } else {
        // exit, if allocate more memory failed
2452
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
2453
               __func__, __LINE__, dbName, sTblName);
2454 2455 2456 2457 2458 2459
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2460
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2461
  }
2462

2463 2464 2465 2466 2467 2468 2469
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2470
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
2471
        char* sTblName, char** childTblNameOfSuperTbl,
2472
        int64_t* childTblCountOfSuperTbl) {
2473

2474 2475
    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
2476
            -1, 0);
2477 2478
}

2479
static int getSuperTableFromServer(TAOS * taos, char* dbName,
2480
        SSuperTable*  superTbls) {
2481

2482
  char command[BUFFER_SIZE] = "\0";
2483
  TAOS_RES * res;
2484 2485
  TAOS_ROW row = NULL;
  int count = 0;
2486

2487
  //get schema use cmd: describe superTblName;
2488
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2489
  res = taos_query(taos, command);
2490 2491 2492
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2493
    taos_free_result(res);
2494 2495 2496 2497 2498 2499
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
2500
  while((row = taos_fetch_row(res)) != NULL) {
2501 2502 2503
    if (0 == count) {
      count++;
      continue;
2504
    }
2505 2506

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2507
      tstrncpy(superTbls->tags[tagIndex].field,
2508 2509
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
2510
      tstrncpy(superTbls->tags[tagIndex].dataType,
2511 2512
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
2513 2514 2515
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
2516 2517
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2518
      tagIndex++;
2519
    } else {
2520
      tstrncpy(superTbls->columns[columnIndex].field,
2521 2522
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
2523
      tstrncpy(superTbls->columns[columnIndex].dataType,
2524 2525
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
2526 2527 2528
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
2529 2530
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2542
/*
2543
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
2544
    //get all child table name use cmd: select tbname from superTblName;
2545 2546 2547
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2548
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2549 2550
      return -1;
    }
2551
    getAllChildNameOfSuperTable(taos, dbName,
2552 2553 2554
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2555
  }
2556
  */
2557 2558 2559
  return 0;
}

H
Haojun Liao 已提交
2560
static int createSuperTable(
2561 2562
        TAOS * taos, char* dbName,
        SSuperTable*  superTbl) {
H
Haojun Liao 已提交
2563

2564
  char command[BUFFER_SIZE] = "\0";
2565

2566 2567 2568 2569 2570
  char cols[STRING_LEN] = "\0";
  int colIndex;
  int len = 0;

  int  lenOfOneRow = 0;
2571 2572 2573

  if (superTbl->columnCount == 0) {
    errorPrint("%s() LN%d, super table column count is %d\n",
2574
            __func__, __LINE__, superTbl->columnCount);
2575 2576 2577 2578 2579
    return -1;
  }

  for (colIndex = 0; colIndex < superTbl->columnCount; colIndex++) {
    char* dataType = superTbl->columns[colIndex].dataType;
2580

2581
    if (strcasecmp(dataType, "BINARY") == 0) {
2582
      len += snprintf(cols + len, STRING_LEN - len,
2583 2584
          ", col%d %s(%d)", colIndex, "BINARY",
          superTbl->columns[colIndex].dataLen);
2585
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2586
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
2587
      len += snprintf(cols + len, STRING_LEN - len,
2588 2589
          ", col%d %s(%d)", colIndex, "NCHAR",
          superTbl->columns[colIndex].dataLen);
2590
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
      lenOfOneRow += 22;
2609
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2610 2611
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
      lenOfOneRow += 42;
2612
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2613 2614 2615 2616
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
      lenOfOneRow += 21;
    } else {
      taos_close(taos);
2617
      errorPrint("%s() LN%d, config error data type : %s\n",
2618
         __func__, __LINE__, dataType);
2619 2620 2621 2622
      exit(-1);
    }
  }

2623 2624
  superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp
  //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbl[j].sTblName, g_Dbs.db[i].superTbl[j].columnCount, lenOfOneRow);
2625 2626

  // save for creating child table
2627 2628 2629
  superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1);
  if (NULL == superTbl->colsOfCreateChildTable) {
    errorPrint("%s() LN%d, Failed when calloc, size:%d",
2630
           __func__, __LINE__, len+1);
2631 2632 2633 2634
    taos_close(taos);
    exit(-1);
  }

2635
  snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
2636
  verbosePrint("%s() LN%d: %s\n",
2637
      __func__, __LINE__, superTbl->colsOfCreateChildTable);
2638

2639 2640
  if (superTbl->tagCount == 0) {
    errorPrint("%s() LN%d, super table tag count is %d\n",
2641
            __func__, __LINE__, superTbl->tagCount);
2642 2643
    return -1;
  }
2644

2645 2646 2647
  char tags[STRING_LEN] = "\0";
  int tagIndex;
  len = 0;
2648

2649 2650 2651 2652
  int lenOfTagOfOneRow = 0;
  len += snprintf(tags + len, STRING_LEN - len, "(");
  for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) {
    char* dataType = superTbl->tags[tagIndex].dataType;
2653

2654 2655
    if (strcasecmp(dataType, "BINARY") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
2656
              "BINARY", superTbl->tags[tagIndex].dataLen);
2657 2658 2659
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
2660
              "NCHAR", superTbl->tags[tagIndex].dataLen);
2661 2662 2663
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2664
              "INT");
2665 2666 2667
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2668
              "BIGINT");
2669 2670 2671
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2672
              "SMALLINT");
2673 2674 2675
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2676
              "TINYINT");
2677 2678 2679
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2680
              "BOOL");
2681 2682 2683
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2684
              "FLOAT");
2685 2686 2687
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2688
              "DOUBLE");
2689 2690 2691
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42;
    } else {
      taos_close(taos);
2692
      errorPrint("%s() LN%d, config error tag type : %s\n",
2693
         __func__, __LINE__, dataType);
2694
      exit(-1);
2695 2696
    }
  }
2697 2698 2699 2700 2701 2702 2703

  len -= 2;
  len += snprintf(tags + len, STRING_LEN - len, ")");

  superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow;

  snprintf(command, BUFFER_SIZE,
2704 2705
          "create table if not exists %s.%s (ts timestamp%s) tags %s",
          dbName, superTbl->sTblName, cols, tags);
2706 2707 2708
  verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command);

  if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2709 2710 2711
      errorPrint( "create supertable %s failed!\n\n",
              superTbl->sTblName);
      return -1;
2712 2713
  }
  debugPrint("create supertable %s success!\n\n", superTbl->sTblName);
2714 2715 2716
  return 0;
}

2717
static int createDatabasesAndStables() {
2718 2719 2720 2721
  TAOS * taos = NULL;
  int    ret = 0;
  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
  if (taos == NULL) {
2722
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2723
    return -1;
2724 2725 2726
  }
  char command[BUFFER_SIZE] = "\0";

2727
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2728 2729
    if (g_Dbs.db[i].drop) {
      sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
2730
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);
2731
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2732 2733 2734 2735
        taos_close(taos);
        return -1;
      }

2736
      int dataLen = 0;
2737
      dataLen += snprintf(command + dataLen,
2738
          BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2739

2740 2741
      if (g_Dbs.db[i].dbCfg.blocks > 0) {
        dataLen += snprintf(command + dataLen,
2742
            BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
2743 2744 2745
      }
      if (g_Dbs.db[i].dbCfg.cache > 0) {
        dataLen += snprintf(command + dataLen,
2746
            BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
2747 2748 2749
      }
      if (g_Dbs.db[i].dbCfg.days > 0) {
        dataLen += snprintf(command + dataLen,
2750
            BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
2751 2752 2753
      }
      if (g_Dbs.db[i].dbCfg.keep > 0) {
        dataLen += snprintf(command + dataLen,
2754
            BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
2755 2756 2757
      }
      if (g_Dbs.db[i].dbCfg.quorum > 1) {
        dataLen += snprintf(command + dataLen,
2758
            BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
2759 2760 2761
      }
      if (g_Dbs.db[i].dbCfg.replica > 0) {
        dataLen += snprintf(command + dataLen,
2762
            BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
2763 2764 2765
      }
      if (g_Dbs.db[i].dbCfg.update > 0) {
        dataLen += snprintf(command + dataLen,
2766
            BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
2767 2768 2769 2770 2771 2772 2773
      }
      //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
      //  dataLen += snprintf(command + dataLen,
      //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
      //}
      if (g_Dbs.db[i].dbCfg.minRows > 0) {
        dataLen += snprintf(command + dataLen,
2774
            BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
2775 2776 2777
      }
      if (g_Dbs.db[i].dbCfg.maxRows > 0) {
        dataLen += snprintf(command + dataLen,
2778
            BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
2779 2780 2781
      }
      if (g_Dbs.db[i].dbCfg.comp > 0) {
        dataLen += snprintf(command + dataLen,
2782
            BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
2783 2784 2785
      }
      if (g_Dbs.db[i].dbCfg.walLevel > 0) {
        dataLen += snprintf(command + dataLen,
2786
            BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
2787 2788 2789
      }
      if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
        dataLen += snprintf(command + dataLen,
2790
            BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
2791 2792 2793
      }
      if (g_Dbs.db[i].dbCfg.fsync > 0) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2794
                " fsync %d", g_Dbs.db[i].dbCfg.fsync);
2795 2796
      }
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
2797 2798
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                      "us", strlen("us")))) {
2799
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2800
                " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
2801 2802 2803 2804 2805 2806 2807 2808 2809
      }

      debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
        taos_close(taos);
        errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
        return -1;
      }
      printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
2810 2811
    }

2812
    debugPrint("%s() LN%d supertbl count:%"PRIu64"\n",
2813
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
2814 2815 2816

    int validStbCount = 0;

S
Shuduo Sang 已提交
2817
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
2818 2819 2820 2821 2822 2823 2824
      sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
              g_Dbs.db[i].superTbls[j].sTblName);
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);

      ret = queryDbExec(taos, command, NO_INSERT_TYPE, true);

      if ((ret != 0) || (g_Dbs.db[i].drop)) {
2825
        ret = createSuperTable(taos, g_Dbs.db[i].dbName,
2826
                &g_Dbs.db[i].superTbls[j]);
2827

2828
        if (0 != ret) {
S
Shuduo Sang 已提交
2829
          errorPrint("create super table %"PRIu64" failed!\n\n", j);
2830
          continue;
2831 2832 2833
        }
      }

2834
      ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
2835
                &g_Dbs.db[i].superTbls[j]);
2836 2837
      if (0 != ret) {
        errorPrint("\nget super table %s.%s info failed!\n\n",
2838
                g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName);
2839
        continue;
2840
      }
2841 2842

      validStbCount ++;
2843
    }
2844 2845

    g_Dbs.db[i].superTblCount = validStbCount;
2846 2847 2848 2849 2850 2851
  }

  taos_close(taos);
  return 0;
}

2852 2853
static void* createTable(void *sarg)
{
2854 2855
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
2856

S
Shuduo Sang 已提交
2857
  uint64_t  lastPrintTime = taosGetTimestampMs();
2858

2859
  int buff_len;
2860
  buff_len = BUFFER_SIZE / 8;
2861

2862
  char *buffer = calloc(buff_len, 1);
2863
  if (buffer == NULL) {
2864
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
2865 2866
    exit(-1);
  }
2867 2868 2869

  int len = 0;
  int batchNum = 0;
2870

2871
  verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n",
2872 2873
          __func__, __LINE__,
          pThreadInfo->start_table_from, pThreadInfo->end_table_to);
2874

2875
  for (uint64_t i = pThreadInfo->start_table_from;
2876
          i <= pThreadInfo->end_table_to; i++) {
2877
    if (0 == g_Dbs.use_metric) {
2878
      snprintf(buffer, buff_len,
2879
              "create table if not exists %s.%s%"PRIu64" %s;",
2880 2881 2882
              pThreadInfo->db_name,
              g_args.tb_prefix, i,
              pThreadInfo->cols);
2883
    } else {
2884 2885
      if (superTblInfo == NULL) {
        errorPrint("%s() LN%d, use metric, but super table info is NULL\n",
2886
                  __func__, __LINE__);
2887
        free(buffer);
2888 2889 2890 2891 2892 2893
        exit(-1);
      } else {
        if (0 == len) {
          batchNum = 0;
          memset(buffer, 0, buff_len);
          len += snprintf(buffer + len,
2894
                  buff_len - len, "create table ");
2895 2896 2897
        }
        char* tagsValBuf = NULL;
        if (0 == superTblInfo->tagSource) {
2898
          tagsValBuf = generateTagVaulesForStb(superTblInfo, i);
2899 2900
        } else {
          tagsValBuf = getTagValueFromTagSample(
2901 2902
                  superTblInfo,
                  i % superTblInfo->tagSampleCount);
2903 2904 2905 2906 2907 2908
        }
        if (NULL == tagsValBuf) {
          free(buffer);
          return NULL;
        }
        len += snprintf(buffer + len,
2909
                buff_len - len,
2910
                "if not exists %s.%s%"PRIu64" using %s.%s tags %s ",
2911 2912 2913
                pThreadInfo->db_name, superTblInfo->childTblPrefix,
                i, pThreadInfo->db_name,
                superTblInfo->sTblName, tagsValBuf);
2914 2915 2916
        free(tagsValBuf);
        batchNum++;
        if ((batchNum < superTblInfo->batchCreateTableNum)
2917 2918
                && ((buff_len - len)
                    >= (superTblInfo->lenOfTagOfOneRow + 256))) {
2919 2920
          continue;
        }
2921 2922 2923 2924
      }
    }

    len = 0;
2925
    verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
2926
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)){
2927
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2928
      free(buffer);
2929 2930 2931
      return NULL;
    }

S
Shuduo Sang 已提交
2932
    uint64_t  currentPrintTime = taosGetTimestampMs();
2933
    if (currentPrintTime - lastPrintTime > 30*1000) {
2934
      printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n",
2935
              pThreadInfo->threadID, pThreadInfo->start_table_from, i);
2936 2937 2938
      lastPrintTime = currentPrintTime;
    }
  }
2939

2940
  if (0 != len) {
2941
    verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer);
2942
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)) {
2943
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2944
    }
2945
  }
2946

2947 2948 2949 2950
  free(buffer);
  return NULL;
}

2951
static int startMultiThreadCreateChildTable(
S
Shuduo Sang 已提交
2952
        char* cols, int threads, uint64_t startFrom, int64_t ntables,
2953
        char* db_name, SSuperTable* superTblInfo) {
2954

2955
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
2956
  threadInfo *infos = calloc(1, threads * sizeof(threadInfo));
2957 2958 2959 2960 2961 2962 2963 2964 2965 2966

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

S
Shuduo Sang 已提交
2967
  int64_t a = ntables / threads;
2968 2969 2970 2971 2972
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

S
Shuduo Sang 已提交
2973
  int64_t b = 0;
2974
  b = ntables % threads;
2975

S
Shuduo Sang 已提交
2976
  for (int i = 0; i < threads; i++) {
2977 2978
    threadInfo *t_info = infos + i;
    t_info->threadID = i;
H
Hui Li 已提交
2979
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
2980
    t_info->superTblInfo = superTblInfo;
2981
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
2982
    t_info->taos = taos_connect(
2983 2984 2985 2986 2987
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
2988
    if (t_info->taos == NULL) {
2989
      errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n",
2990
         __func__, __LINE__, taos_errstr(NULL));
2991
      free(pids);
2992
      free(infos);
2993 2994
      return -1;
    }
2995 2996 2997 2998 2999

    t_info->start_table_from = startFrom;
    t_info->ntables = i<b?a+1:a;
    t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
    startFrom = t_info->end_table_to + 1;
3000
    t_info->use_metric = true;
3001
    t_info->cols = cols;
3002
    t_info->minDelay = UINT64_MAX;
3003 3004
    pthread_create(pids + i, NULL, createTable, t_info);
  }
3005

3006 3007 3008 3009 3010 3011 3012 3013 3014 3015
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
    threadInfo *t_info = infos + i;
    taos_close(t_info->taos);
  }

  free(pids);
3016
  free(infos);
3017 3018

  return 0;
3019 3020
}

3021
static void createChildTables() {
3022 3023
    char tblColsBuf[MAX_SQL_SIZE];
    int len;
3024

3025
  for (int i = 0; i < g_Dbs.dbCount; i++) {
3026 3027
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
3028
          // with super table
S
Shuduo Sang 已提交
3029
        for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
3030
          if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
3031
                || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
3032 3033
            continue;
          }
3034

3035
          verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
3036
                  g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
S
Shuduo Sang 已提交
3037
          uint64_t startFrom = 0;
3038 3039
          g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;

3040
          verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n",
3041
                  __func__, __LINE__, g_totalChildTables, startFrom);
3042
          startMultiThreadCreateChildTable(
3043 3044 3045 3046 3047
                g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
                g_Dbs.threadCountByCreateTbl,
                startFrom,
                g_Dbs.db[i].superTbls[j].childTblCount,
                g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
3048
        }
3049 3050
      }
    } else {
3051 3052
      // normal table
      len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
H
Haojun Liao 已提交
3053
      for (int j = 0; j < g_args.num_of_CPR; j++) {
3054 3055 3056 3057 3058 3059 3060 3061 3062 3063
          if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
                  || (strncasecmp(g_args.datatype[j],
                      "NCHAR", strlen("NCHAR")) == 0)) {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                      ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary);
          } else {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                      ", COL%d %s", j, g_args.datatype[j]);
          }
          len = strlen(tblColsBuf);
3064
      }
3065

3066
      snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");
3067

3068
      verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n",
3069 3070
              __func__, __LINE__,
              g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
3071
      startMultiThreadCreateChildTable(
3072 3073 3074 3075 3076 3077
            tblColsBuf,
            g_Dbs.threadCountByCreateTbl,
            0,
            g_args.num_of_tables,
            g_Dbs.db[i].dbName,
            NULL);
3078
    }
3079 3080 3081 3082 3083 3084
  }
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3085
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
3086 3087 3088
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
3089

3090 3091
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
3092
    printf("Failed to open tags file: %s, reason:%s\n",
3093
            superTblInfo->tagsFile, strerror(errno));
3094 3095 3096 3097 3098 3099 3100
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
3101

3102 3103 3104 3105 3106 3107 3108 3109 3110
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

3111
  while((readLen = tgetline(&line, &n, fp)) != -1) {
3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
3124
      char *tmp = realloc(tagDataBuf,
3125
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
3126 3127 3128
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
3129
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
3130
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

3150
#if 0
3151 3152 3153 3154
int readSampleFromJsonFileToMem(SSuperTable  * superTblInfo) {
  // TODO
  return 0;
}
3155
#endif
3156 3157 3158 3159

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3160
static int readSampleFromCsvFileToMem(
3161
        SSuperTable* superTblInfo) {
3162 3163 3164 3165
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
3166

3167 3168
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
3169 3170 3171
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
              superTblInfo->sampleFile, strerror(errno));
      return -1;
3172
  }
3173

3174
  assert(superTblInfo->sampleDataBuf);
3175
  memset(superTblInfo->sampleDataBuf, 0,
3176
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
3177
  while(1) {
3178
    readLen = tgetline(&line, &n, fp);
3179 3180
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
3181
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
3182
                superTblInfo->sampleFile, strerror(errno));
3183
        fclose(fp);
3184 3185 3186 3187
        return -1;
      }
      continue;
    }
3188

3189 3190 3191 3192 3193 3194 3195 3196 3197
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
3198
      printf("sample row len[%d] overflow define schema len[%"PRIu64"], so discard this row\n",
3199
              (int32_t)readLen, superTblInfo->lenOfOneRow);
3200 3201 3202
      continue;
    }

3203
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
3204
          line, readLen);
3205 3206 3207 3208 3209 3210 3211
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

3212
  fclose(fp);
3213 3214 3215 3216
  tmfree(line);
  return 0;
}

3217
static bool getColumnAndTagTypeFromInsertJsonFile(
3218
        cJSON* stbInfo, SSuperTable* superTbls) {
3219
  bool  ret = false;
3220

3221
  // columns
3222 3223
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
3224
    printf("ERROR: failed to read json, columns not found\n");
3225 3226 3227 3228 3229 3230
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
3231

3232
  int columnSize = cJSON_GetArraySize(columns);
3233
  if ((columnSize + 1/* ts */) > MAX_COLUMN_COUNT) {
3234
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
3235
            __func__, __LINE__, MAX_COLUMN_COUNT);
3236 3237 3238 3239 3240 3241
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
3242

3243
  //superTbls->columnCount = columnSize;
3244 3245 3246 3247 3248 3249 3250
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
3251
      count = countObj->valueint;
3252
    } else if (countObj && countObj->type != cJSON_Number) {
3253
      errorPrint("%s() LN%d, failed to read json, column count not found\n",
3254
          __func__, __LINE__);
3255 3256 3257 3258 3259
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3260
    // column info
3261 3262
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
3263 3264 3265
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d: failed to read json, column type not found\n",
3266
          __func__, __LINE__);
3267 3268
      goto PARSE_OVER;
    }
3269 3270
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3271

3272 3273
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3274
      columnCase.dataLen = dataLen->valueint;
3275
    } else if (dataLen && dataLen->type != cJSON_Number) {
3276
      debugPrint("%s() LN%d: failed to read json, column len not found\n",
3277
          __func__, __LINE__);
3278 3279 3280 3281
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
3282

3283
    for (int n = 0; n < count; ++n) {
3284
      tstrncpy(superTbls->columns[index].dataType,
3285
              columnCase.dataType, MAX_TB_NAME_SIZE);
3286
      superTbls->columns[index].dataLen = columnCase.dataLen;
3287 3288
      index++;
    }
3289
  }
3290

3291 3292
  if ((index + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n",
3293
            __func__, __LINE__, MAX_COLUMN_COUNT);
3294 3295 3296
    goto PARSE_OVER;
  }

3297
  superTbls->columnCount = index;
3298

3299 3300
  count = 1;
  index = 0;
3301
  // tags
3302 3303
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
3304
    errorPrint("%s() LN%d, failed to read json, tags not found\n",
3305
        __func__, __LINE__);
3306 3307
    goto PARSE_OVER;
  }
3308

3309 3310
  int tagSize = cJSON_GetArraySize(tags);
  if (tagSize > MAX_TAG_COUNT) {
3311
    errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n",
3312
        __func__, __LINE__, MAX_TAG_COUNT);
3313 3314
    goto PARSE_OVER;
  }
3315 3316

  //superTbls->tagCount = tagSize;
3317 3318 3319
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
3320

3321 3322 3323
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
3324
      count = countObj->valueint;
3325
    } else if (countObj && countObj->type != cJSON_Number) {
3326
      printf("ERROR: failed to read json, column count not found\n");
3327 3328 3329 3330 3331
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3332
    // column info
3333 3334
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
3335 3336 3337
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d, failed to read json, tag type not found\n",
3338
          __func__, __LINE__);
3339 3340
      goto PARSE_OVER;
    }
3341
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3342

3343 3344
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3345
      columnCase.dataLen = dataLen->valueint;
3346
    } else if (dataLen && dataLen->type != cJSON_Number) {
3347
      errorPrint("%s() LN%d, failed to read json, column len not found\n",
3348
          __func__, __LINE__);
3349 3350 3351
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
3352 3353
    }

3354
    for (int n = 0; n < count; ++n) {
3355
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType,
3356
          MAX_TB_NAME_SIZE);
3357
      superTbls->tags[index].dataLen = columnCase.dataLen;
3358 3359
      index++;
    }
3360
  }
3361 3362

  if (index > MAX_TAG_COUNT) {
3363
    errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n",
3364
        __func__, __LINE__, MAX_TAG_COUNT);
3365 3366 3367
    goto PARSE_OVER;
  }

3368 3369
  superTbls->tagCount = index;

3370 3371
  if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n",
3372
        __func__, __LINE__, MAX_COLUMN_COUNT);
3373 3374
    goto PARSE_OVER;
  }
3375 3376
  ret = true;

3377
PARSE_OVER:
3378 3379 3380 3381 3382 3383 3384 3385
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3386
    tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3387 3388 3389 3390
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3391
    tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE);
3392
  } else if (!host) {
3393
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3394
  } else {
3395
    printf("ERROR: failed to read json, host not found\n");
3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_Dbs.port = port->valueint;
  } else if (!port) {
    g_Dbs.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3408
    tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE);
3409
  } else if (!user) {
3410
    tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE);
3411 3412 3413 3414
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3415
    tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE);
3416
  } else if (!password) {
3417
    tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE);
3418 3419 3420 3421
  }

  cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
  if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
3422
    tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
3423
  } else if (!resultfile) {
3424
    tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
3425 3426 3427 3428 3429 3430 3431 3432
  }

  cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
  if (threads && threads->type == cJSON_Number) {
    g_Dbs.threadCount = threads->valueint;
  } else if (!threads) {
    g_Dbs.threadCount = 1;
  } else {
3433
    printf("ERROR: failed to read json, threads not found\n");
3434
    goto PARSE_OVER;
3435 3436
  }

3437 3438 3439 3440
  cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
  if (threads2 && threads2->type == cJSON_Number) {
    g_Dbs.threadCountByCreateTbl = threads2->valueint;
  } else if (!threads2) {
sangshuduo's avatar
sangshuduo 已提交
3441
    g_Dbs.threadCountByCreateTbl = 1;
3442
  } else {
3443
    errorPrint("%s() LN%d, failed to read json, threads2 not found\n",
3444
            __func__, __LINE__);
3445
    goto PARSE_OVER;
3446
  }
3447

3448 3449
  cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
  if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
3450 3451 3452 3453 3454
    if (gInsertInterval->valueint <0) {
      errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n",
            __func__, __LINE__);
      goto PARSE_OVER;
    }
3455 3456 3457 3458
    g_args.insert_interval = gInsertInterval->valueint;
  } else if (!gInsertInterval) {
    g_args.insert_interval = 0;
  } else {
3459
    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
3460
        __func__, __LINE__);
3461 3462
    goto PARSE_OVER;
  }
3463

3464 3465
  cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows");
  if (interlaceRows && interlaceRows->type == cJSON_Number) {
3466 3467 3468 3469 3470 3471
    if (interlaceRows->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;

    }
3472
    g_args.interlace_rows = interlaceRows->valueint;
3473 3474 3475

    // rows per table need be less than insert batch
    if (g_args.interlace_rows > g_args.num_of_RPR) {
3476
      printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
3477
              g_args.interlace_rows, g_args.num_of_RPR);
3478
      printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
3479
              g_args.num_of_RPR);
3480
      prompt();
3481 3482
      g_args.interlace_rows = g_args.num_of_RPR;
    }
3483
  } else if (!interlaceRows) {
3484
    g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3485
  } else {
3486
    errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
3487
        __func__, __LINE__);
3488
    goto PARSE_OVER;
3489
  }
3490

3491 3492
  cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
  if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
3493 3494 3495 3496 3497
    if (maxSqlLen->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
    }
3498 3499
    g_args.max_sql_len = maxSqlLen->valueint;
  } else if (!maxSqlLen) {
sangshuduo's avatar
sangshuduo 已提交
3500
    g_args.max_sql_len = (1024*1024);
3501
  } else {
3502
    errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
3503
        __func__, __LINE__);
3504 3505 3506 3507 3508
    goto PARSE_OVER;
  }

  cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
  if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
3509
    if (numRecPerReq->valueint <= 0) {
3510 3511 3512
      errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
3513 3514
    } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) {
      numRecPerReq->valueint = MAX_RECORDS_PER_REQ;
3515
    }
3516 3517
    g_args.num_of_RPR = numRecPerReq->valueint;
  } else if (!numRecPerReq) {
3518
    g_args.num_of_RPR = MAX_RECORDS_PER_REQ;
3519
  } else {
3520
    errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n",
3521
        __func__, __LINE__);
3522 3523 3524
    goto PARSE_OVER;
  }

3525
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3526
  if (answerPrompt
3527 3528
          && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3529 3530 3531 3532 3533 3534 3535 3536
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
    } else {
      g_args.answer_yes = false;
    }
  } else if (!answerPrompt) {
3537
    g_args.answer_yes = false;
3538
  } else {
3539
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
3540
    goto PARSE_OVER;
3541
  }
3542 3543 3544

  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (!dbs || dbs->type != cJSON_Array) {
3545
    printf("ERROR: failed to read json, databases not found\n");
3546 3547 3548 3549 3550
    goto PARSE_OVER;
  }

  int dbSize = cJSON_GetArraySize(dbs);
  if (dbSize > MAX_DB_COUNT) {
3551
    errorPrint(
3552 3553
            "ERROR: failed to read json, databases size overflow, max database is %d\n",
            MAX_DB_COUNT);
3554 3555 3556 3557 3558 3559 3560 3561
    goto PARSE_OVER;
  }

  g_Dbs.dbCount = dbSize;
  for (int i = 0; i < dbSize; ++i) {
    cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
    if (dbinfos == NULL) continue;

3562
    // dbinfo
3563 3564
    cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
    if (!dbinfo || dbinfo->type != cJSON_Object) {
3565
      printf("ERROR: failed to read json, dbinfo not found\n");
3566 3567
      goto PARSE_OVER;
    }
3568

3569 3570
    cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
    if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
3571
      printf("ERROR: failed to read json, db name not found\n");
3572 3573
      goto PARSE_OVER;
    }
3574
    tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE);
3575 3576 3577

    cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
    if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
3578 3579
      if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) {
        g_Dbs.db[i].drop = true;
3580
      } else {
3581
        g_Dbs.db[i].drop = false;
3582
      }
3583
    } else if (!drop) {
3584
      g_Dbs.db[i].drop = g_args.drop_database;
3585
    } else {
3586
      errorPrint("%s() LN%d, failed to read json, drop input mistake\n",
3587
              __func__, __LINE__);
3588 3589 3590 3591
      goto PARSE_OVER;
    }

    cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
3592
    if (precision && precision->type == cJSON_String
3593
            && precision->valuestring != NULL) {
3594
      tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
3595
              MAX_DB_NAME_SIZE);
3596
    } else if (!precision) {
3597
      //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
3598 3599
      memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE);
    } else {
3600
      printf("ERROR: failed to read json, precision not found\n");
3601 3602 3603 3604 3605 3606 3607 3608 3609
      goto PARSE_OVER;
    }

    cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
    if (update && update->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.update = update->valueint;
    } else if (!update) {
      g_Dbs.db[i].dbCfg.update = -1;
    } else {
3610
      printf("ERROR: failed to read json, update not found\n");
3611 3612 3613 3614 3615 3616 3617 3618 3619
      goto PARSE_OVER;
    }

    cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
    if (replica && replica->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.replica = replica->valueint;
    } else if (!replica) {
      g_Dbs.db[i].dbCfg.replica = -1;
    } else {
3620
      printf("ERROR: failed to read json, replica not found\n");
3621 3622 3623 3624 3625 3626 3627 3628 3629
      goto PARSE_OVER;
    }

    cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
    if (keep && keep->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.keep = keep->valueint;
    } else if (!keep) {
      g_Dbs.db[i].dbCfg.keep = -1;
    } else {
3630 3631
     printf("ERROR: failed to read json, keep not found\n");
     goto PARSE_OVER;
3632
    }
3633

3634 3635 3636 3637 3638 3639
    cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
    if (days && days->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.days = days->valueint;
    } else if (!days) {
      g_Dbs.db[i].dbCfg.days = -1;
    } else {
3640 3641
     printf("ERROR: failed to read json, days not found\n");
     goto PARSE_OVER;
3642
    }
3643

3644 3645 3646 3647 3648 3649
    cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
    if (cache && cache->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cache = cache->valueint;
    } else if (!cache) {
      g_Dbs.db[i].dbCfg.cache = -1;
    } else {
3650 3651
     printf("ERROR: failed to read json, cache not found\n");
     goto PARSE_OVER;
3652
    }
3653

3654 3655 3656 3657 3658 3659
    cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
    if (blocks && blocks->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
    } else if (!blocks) {
      g_Dbs.db[i].dbCfg.blocks = -1;
    } else {
3660 3661
     printf("ERROR: failed to read json, block not found\n");
     goto PARSE_OVER;
3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677
    }

    //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
    //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
    //} else if (!maxtablesPerVnode) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
    //} else {
    // printf("failed to read json, maxtablesPerVnode not found");
    // goto PARSE_OVER;
    //}

    cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
    if (minRows && minRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
    } else if (!minRows) {
3678
      g_Dbs.db[i].dbCfg.minRows = 0;    // 0 means default
3679
    } else {
3680 3681
     printf("ERROR: failed to read json, minRows not found\n");
     goto PARSE_OVER;
3682 3683 3684 3685 3686 3687
    }

    cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
    if (maxRows && maxRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
    } else if (!maxRows) {
3688
      g_Dbs.db[i].dbCfg.maxRows = 0;    // 0 means default
3689
    } else {
3690 3691
     printf("ERROR: failed to read json, maxRows not found\n");
     goto PARSE_OVER;
3692 3693 3694 3695 3696 3697 3698 3699
    }

    cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
    if (comp && comp->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.comp = comp->valueint;
    } else if (!comp) {
      g_Dbs.db[i].dbCfg.comp = -1;
    } else {
3700 3701
     printf("ERROR: failed to read json, comp not found\n");
     goto PARSE_OVER;
3702 3703 3704 3705 3706 3707 3708 3709
    }

    cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
    if (walLevel && walLevel->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
    } else if (!walLevel) {
      g_Dbs.db[i].dbCfg.walLevel = -1;
    } else {
3710 3711
     printf("ERROR: failed to read json, walLevel not found\n");
     goto PARSE_OVER;
3712 3713
    }

3714 3715 3716 3717 3718 3719
    cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
    if (cacheLast && cacheLast->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
    } else if (!cacheLast) {
      g_Dbs.db[i].dbCfg.cacheLast = -1;
    } else {
3720 3721
     printf("ERROR: failed to read json, cacheLast not found\n");
     goto PARSE_OVER;
3722 3723
    }

3724 3725 3726 3727
    cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
    if (quorum && quorum->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
    } else if (!quorum) {
3728
      g_Dbs.db[i].dbCfg.quorum = 1;
3729
    } else {
3730 3731
     printf("failed to read json, quorum input mistake");
     goto PARSE_OVER;
3732 3733 3734 3735 3736 3737 3738 3739
    }

    cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
    if (fsync && fsync->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
    } else if (!fsync) {
      g_Dbs.db[i].dbCfg.fsync = -1;
    } else {
3740
      errorPrint("%s() LN%d, failed to read json, fsync input mistake\n",
3741
              __func__, __LINE__);
3742
      goto PARSE_OVER;
3743
    }
3744

3745
    // super_talbes
3746 3747
    cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
    if (!stables || stables->type != cJSON_Array) {
3748
      errorPrint("%s() LN%d, failed to read json, super_tables not found\n",
3749
              __func__, __LINE__);
3750
      goto PARSE_OVER;
3751 3752
    }

3753 3754
    int stbSize = cJSON_GetArraySize(stables);
    if (stbSize > MAX_SUPER_TABLE_COUNT) {
3755
      errorPrint(
3756 3757
              "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n",
              __func__, __LINE__, MAX_SUPER_TABLE_COUNT);
3758 3759 3760 3761 3762 3763 3764
      goto PARSE_OVER;
    }

    g_Dbs.db[i].superTblCount = stbSize;
    for (int j = 0; j < stbSize; ++j) {
      cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
      if (stbInfo == NULL) continue;
3765 3766

      // dbinfo
3767 3768
      cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
      if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) {
3769
        errorPrint("%s() LN%d, failed to read json, stb name not found\n",
3770
                __func__, __LINE__);
3771 3772
        goto PARSE_OVER;
      }
3773
      tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE);
3774

3775 3776
      cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
      if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
3777
        printf("ERROR: failed to read json, childtable_prefix not found\n");
3778 3779
        goto PARSE_OVER;
      }
3780
      tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE);
3781 3782

      cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null
3783
      if (autoCreateTbl
3784 3785
              && autoCreateTbl->type == cJSON_String
              && autoCreateTbl->valuestring != NULL) {
3786 3787 3788 3789 3790 3791 3792 3793 3794 3795
        if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
        } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        } else {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        }
      } else if (!autoCreateTbl) {
        g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      } else {
3796
        printf("ERROR: failed to read json, auto_create_table not found\n");
3797 3798
        goto PARSE_OVER;
      }
3799

3800 3801 3802 3803
      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
      } else if (!batchCreateTbl) {
3804
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
3805
      } else {
3806
        printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
3807
        goto PARSE_OVER;
3808
      }
3809 3810

      cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
3811
      if (childTblExists
3812 3813
              && childTblExists->type == cJSON_String
              && childTblExists->valuestring != NULL) {
3814 3815
        if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3))
            && (g_Dbs.db[i].drop == false)) {
3816
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
3817
        } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2)
3818
              || (g_Dbs.db[i].drop == true))) {
3819 3820 3821 3822 3823 3824 3825
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        } else {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        }
      } else if (!childTblExists) {
        g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
      } else {
3826
        errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n",
3827
                __func__, __LINE__);
3828 3829
        goto PARSE_OVER;
      }
3830

3831 3832
      cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
      if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
3833
        errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n",
3834
                __func__, __LINE__);
3835 3836 3837 3838 3839
        goto PARSE_OVER;
      }
      g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

      cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
3840
      if (dataSource && dataSource->type == cJSON_String
3841
              && dataSource->valuestring != NULL) {
3842
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
3843
                dataSource->valuestring, MAX_DB_NAME_SIZE);
3844
      } else if (!dataSource) {
3845
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE);
3846
      } else {
3847
        errorPrint("%s() LN%d, failed to read json, data_source not found\n",
3848
            __func__, __LINE__);
3849 3850 3851
        goto PARSE_OVER;
      }

3852
      cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt
3853
      if (insertMode && insertMode->type == cJSON_String
3854
              && insertMode->valuestring != NULL) {
3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865
        if (0 == strcasecmp(insertMode->valuestring, "taosc")) {
            g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE;
        } else if (0 == strcasecmp(insertMode->valuestring, "rest")) {
            g_Dbs.db[i].superTbls[j].insertMode = REST_IFACE;
        } else if (0 == strcasecmp(insertMode->valuestring, "stmt")) {
            g_Dbs.db[i].superTbls[j].insertMode = STMT_IFACE;
        } else {
            errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n",
                    __func__, __LINE__, insertMode->valuestring);
            goto PARSE_OVER;
        }
3866
      } else if (!insertMode) {
3867
        g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE;
3868
      } else {
3869
        errorPrint("%s", "failed to read json, insert_mode not found\n");
3870 3871 3872
        goto PARSE_OVER;
      }

3873
      cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
3874 3875
      if ((childTbl_limit) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3876
        if (childTbl_limit->type != cJSON_Number) {
3877 3878
            printf("ERROR: failed to read json, childtable_limit\n");
            goto PARSE_OVER;
3879 3880 3881
        }
        g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
      } else {
3882
        g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid.
3883 3884 3885
      }

      cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
3886 3887
      if ((childTbl_offset) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3888
        if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) {
3889 3890
            printf("ERROR: failed to read json, childtable_offset\n");
            goto PARSE_OVER;
3891 3892 3893 3894 3895 3896
        }
        g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblOffset = 0;
      }

3897 3898
      cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
      if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
3899
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
3900
                ts->valuestring, MAX_DB_NAME_SIZE);
3901
      } else if (!ts) {
3902
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
3903
                "now", MAX_DB_NAME_SIZE);
3904
      } else {
3905
        printf("ERROR: failed to read json, start_timestamp not found\n");
3906 3907
        goto PARSE_OVER;
      }
3908

3909 3910 3911 3912
      cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
      if (timestampStep && timestampStep->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
      } else if (!timestampStep) {
3913
        g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
3914
      } else {
3915
        printf("ERROR: failed to read json, timestamp_step not found\n");
3916 3917 3918 3919
        goto PARSE_OVER;
      }

      cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
3920
      if (sampleFormat && sampleFormat->type
3921
              == cJSON_String && sampleFormat->valuestring != NULL) {
3922
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
3923
                sampleFormat->valuestring, MAX_DB_NAME_SIZE);
3924
      } else if (!sampleFormat) {
3925
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE);
3926
      } else {
3927
        printf("ERROR: failed to read json, sample_format not found\n");
3928
        goto PARSE_OVER;
3929
      }
3930

3931
      cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
3932 3933
      if (sampleFile && sampleFile->type == cJSON_String
          && sampleFile->valuestring != NULL) {
3934
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
3935
                sampleFile->valuestring, MAX_FILE_NAME_LEN);
3936 3937 3938
      } else if (!sampleFile) {
        memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
      } else {
3939
        printf("ERROR: failed to read json, sample_file not found\n");
3940
        goto PARSE_OVER;
3941
      }
3942

3943 3944
      cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
      if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) {
3945
        tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
3946
                tagsFile->valuestring, MAX_FILE_NAME_LEN);
3947 3948 3949 3950 3951 3952 3953 3954 3955
        if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
          g_Dbs.db[i].superTbls[j].tagSource = 0;
        } else {
          g_Dbs.db[i].superTbls[j].tagSource = 1;
        }
      } else if (!tagsFile) {
        memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
        g_Dbs.db[i].superTbls[j].tagSource = 0;
      } else {
3956
        printf("ERROR: failed to read json, tags_file not found\n");
3957 3958
        goto PARSE_OVER;
      }
3959

sangshuduo's avatar
sangshuduo 已提交
3960 3961 3962
      cJSON* stbMaxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
      if (stbMaxSqlLen && stbMaxSqlLen->type == cJSON_Number) {
        int32_t len = stbMaxSqlLen->valueint;
3963 3964
        if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
          len = TSDB_MAX_ALLOWED_SQL_LEN;
3965 3966
        } else if (len < 5) {
          len = 5;
3967
        }
3968 3969
        g_Dbs.db[i].superTbls[j].maxSqlLen = len;
      } else if (!maxSqlLen) {
3970
        g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len;
3971
      } else {
sangshuduo's avatar
sangshuduo 已提交
3972
        errorPrint("%s() LN%d, failed to read json, stbMaxSqlLen input mistake\n",
3973
            __func__, __LINE__);
3974
        goto PARSE_OVER;
3975
      }
3976
/*
3977 3978 3979 3980 3981
      cJSON *multiThreadWriteOneTbl =
          cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
      if (multiThreadWriteOneTbl
              && multiThreadWriteOneTbl->type == cJSON_String
              && multiThreadWriteOneTbl->valuestring != NULL) {
3982 3983 3984 3985
        if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
        } else {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
3986
        }
3987 3988 3989
      } else if (!multiThreadWriteOneTbl) {
        g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
      } else {
3990
        printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
3991 3992
        goto PARSE_OVER;
      }
3993
*/
sangshuduo's avatar
sangshuduo 已提交
3994 3995 3996
      cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows");
      if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) {
        if (stbInterlaceRows->valueint < 0) {
3997 3998 3999 4000
          errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n",
            __func__, __LINE__);
          goto PARSE_OVER;
        }
sangshuduo's avatar
sangshuduo 已提交
4001
        g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint;
4002 4003
        // rows per table need be less than insert batch
        if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) {
4004
          printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
4005
                  i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR);
4006
          printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
4007
                  g_args.num_of_RPR);
4008 4009 4010 4011
          if (!g_args.answer_yes) {
            printf("        press Enter key to continue or Ctrl-C to stop.");
            (void)getchar();
          }
4012 4013
          g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR;
        }
sangshuduo's avatar
sangshuduo 已提交
4014
      } else if (!stbInterlaceRows) {
4015
        g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
4016
      } else {
4017
        errorPrint(
4018 4019
                "%s() LN%d, failed to read json, interlace rows input mistake\n",
                __func__, __LINE__);
4020
        goto PARSE_OVER;
4021
      }
4022 4023 4024

      cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
      if (disorderRatio && disorderRatio->type == cJSON_Number) {
4025 4026 4027 4028 4029 4030
        if (disorderRatio->valueint > 50)
          disorderRatio->valueint = 50;

        if (disorderRatio->valueint < 0)
          disorderRatio->valueint = 0;

4031 4032 4033 4034
        g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
      } else if (!disorderRatio) {
        g_Dbs.db[i].superTbls[j].disorderRatio = 0;
      } else {
4035
        printf("ERROR: failed to read json, disorderRatio not found\n");
4036
        goto PARSE_OVER;
4037
      }
4038 4039 4040 4041 4042 4043 4044

      cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
      if (disorderRange && disorderRange->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
      } else if (!disorderRange) {
        g_Dbs.db[i].superTbls[j].disorderRange = 1000;
      } else {
4045
        printf("ERROR: failed to read json, disorderRange not found\n");
4046 4047
        goto PARSE_OVER;
      }
4048

4049 4050
      cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
      if (insertRows && insertRows->type == cJSON_Number) {
4051 4052 4053 4054 4055
        if (insertRows->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4056 4057 4058 4059
        g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
      } else if (!insertRows) {
        g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
      } else {
4060
        errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
4061
                __func__, __LINE__);
4062 4063 4064 4065 4066 4067
        goto PARSE_OVER;
      }

      cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
      if (insertInterval && insertInterval->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
4068 4069 4070 4071 4072
        if (insertInterval->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4073
      } else if (!insertInterval) {
4074
        verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n",
4075
                __func__, __LINE__, g_args.insert_interval);
4076 4077
        g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
      } else {
4078
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
4079
                __func__, __LINE__);
4080 4081 4082
        goto PARSE_OVER;
      }

4083
      int retVal = getColumnAndTagTypeFromInsertJsonFile(
4084
              stbInfo, &g_Dbs.db[i].superTbls[j]);
4085 4086
      if (false == retVal) {
        goto PARSE_OVER;
4087 4088
      }
    }
4089 4090 4091 4092
  }

  ret = true;

4093
PARSE_OVER:
4094 4095 4096 4097 4098 4099 4100 4101
  return ret;
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
4102
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
4103 4104 4105 4106
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
4107
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE);
4108
  } else if (!host) {
4109
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
4110
  } else {
4111
    printf("ERROR: failed to read json, host not found\n");
4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
4124
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE);
4125
  } else if (!user) {
4126
    tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ;
4127 4128 4129 4130
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
4131
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE);
4132
  } else if (!password) {
4133
    tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);;
4134
  }
H
hzcheng 已提交
4135

4136
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
4137
  if (answerPrompt && answerPrompt->type == cJSON_String
4138
          && answerPrompt->valuestring != NULL) {
4139 4140 4141 4142
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
4143
    } else {
4144
      g_args.answer_yes = false;
H
hzcheng 已提交
4145
    }
4146 4147 4148
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
4149
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
4150
    goto PARSE_OVER;
4151
  }
4152

4153 4154
  cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
  if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
4155
    if (gQueryTimes->valueint <= 0) {
4156 4157
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4158 4159
      goto PARSE_OVER;
    }
4160 4161 4162 4163
    g_args.query_times = gQueryTimes->valueint;
  } else if (!gQueryTimes) {
    g_args.query_times = 1;
  } else {
4164
    errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4165
        __func__, __LINE__);
4166 4167 4168
    goto PARSE_OVER;
  }

4169 4170
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
4171
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE);
4172
  } else if (!dbs) {
4173
    printf("ERROR: failed to read json, databases not found\n");
4174
    goto PARSE_OVER;
H
hzcheng 已提交
4175
  }
4176 4177 4178

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
4179
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
4180
  } else if (!queryMode) {
4181
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
4182
  } else {
4183
    printf("ERROR: failed to read json, query_mode not found\n");
4184
    goto PARSE_OVER;
H
hzcheng 已提交
4185
  }
4186

4187
  // specified_table_query
4188 4189
  cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!specifiedQuery) {
4190
    g_queryInfo.specifiedQueryInfo.concurrent = 1;
4191 4192
    g_queryInfo.specifiedQueryInfo.sqlCount = 0;
  } else if (specifiedQuery->type != cJSON_Object) {
4193
    printf("ERROR: failed to read json, super_table_query not found\n");
4194
    goto PARSE_OVER;
4195
  } else {
4196 4197 4198 4199 4200
    cJSON* queryInterval = cJSON_GetObjectItem(specifiedQuery, "query_interval");
    if (queryInterval && queryInterval->type == cJSON_Number) {
      g_queryInfo.specifiedQueryInfo.queryInterval = queryInterval->valueint;
    } else if (!queryInterval) {
      g_queryInfo.specifiedQueryInfo.queryInterval = 0;
4201 4202
    }

4203
    cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery,
4204
        "query_times");
4205
    if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
4206 4207
      if (specifiedQueryTimes->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4208
          __func__, __LINE__, specifiedQueryTimes->valueint);
4209 4210 4211
        goto PARSE_OVER;

      }
4212 4213 4214 4215
      g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint;
    } else if (!specifiedQueryTimes) {
      g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times;
    } else {
4216
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4217
          __func__, __LINE__);
4218
      goto PARSE_OVER;
4219
    }
4220

4221
    cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent");
4222
    if (concurrent && concurrent->type == cJSON_Number) {
4223 4224 4225 4226
      if (concurrent->valueint <= 0) {
        errorPrint("%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n",
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount,
4227
              g_queryInfo.specifiedQueryInfo.concurrent);
4228 4229
        goto PARSE_OVER;
      }
4230
      g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint;
4231
    } else if (!concurrent) {
4232
      g_queryInfo.specifiedQueryInfo.concurrent = 1;
4233
    }
4234

4235 4236 4237 4238 4239 4240 4241
    cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode");
    if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String
        && specifiedAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE;
4242
      } else {
4243
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4244
            __func__, __LINE__);
4245 4246 4247
        goto PARSE_OVER;
      }
    } else {
4248
      g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
4249
    }
4250

4251
    cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval");
4252
    if (interval && interval->type == cJSON_Number) {
4253
      g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint;
4254
    } else if (!interval) {
4255 4256
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4257
      g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000;
4258
    }
4259

4260
    cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart");
4261
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
4262
      if (0 == strcmp("yes", restart->valuestring)) {
4263
        g_queryInfo.specifiedQueryInfo.subscribeRestart = 1;
4264
      } else if (0 == strcmp("no", restart->valuestring)) {
4265
        g_queryInfo.specifiedQueryInfo.subscribeRestart = 0;
4266
      } else {
4267
        printf("ERROR: failed to read json, subscribe restart error\n");
4268 4269 4270
        goto PARSE_OVER;
      }
    } else {
4271
      g_queryInfo.specifiedQueryInfo.subscribeRestart = 1;
4272
    }
4273

4274
    cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress");
4275
    if (keepProgress
4276 4277
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
4278
      if (0 == strcmp("yes", keepProgress->valuestring)) {
4279
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1;
4280
      } else if (0 == strcmp("no", keepProgress->valuestring)) {
4281
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4282
      } else {
4283
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4284 4285 4286
        goto PARSE_OVER;
      }
    } else {
4287
      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4288 4289
    }

4290
    // sqls
4291
    cJSON* superSqls = cJSON_GetObjectItem(specifiedQuery, "sqls");
4292
    if (!superSqls) {
4293
      g_queryInfo.specifiedQueryInfo.sqlCount = 0;
4294
    } else if (superSqls->type != cJSON_Array) {
4295
      errorPrint("%s() LN%d, failed to read json, super sqls not found\n",
4296
          __func__, __LINE__);
4297
      goto PARSE_OVER;
4298
    } else {
4299 4300
      int superSqlSize = cJSON_GetArraySize(superSqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4301
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
4302
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4303 4304
        goto PARSE_OVER;
      }
4305

4306
      g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize;
4307 4308 4309
      for (int j = 0; j < superSqlSize; ++j) {
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
        if (sql == NULL) continue;
4310

4311 4312
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
4313
          printf("ERROR: failed to read json, sql not found\n");
4314 4315
          goto PARSE_OVER;
        }
4316
        tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
H
hzcheng 已提交
4317

4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329
        cJSON* resubAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume");
        if (resubAfterConsume
                && resubAfterConsume->type == cJSON_Number) {
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j]
                = resubAfterConsume->valueint;
        } else if (!resubAfterConsume) {
            //printf("failed to read json, subscribe interval no found\n");
            //goto PARSE_OVER;
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = 1;
        }

4330 4331
        cJSON *result = cJSON_GetObjectItem(sql, "result");
        if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) {
4332
          tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN);
4333
        } else if (NULL == result) {
4334
          memset(g_queryInfo.specifiedQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4335
        } else {
4336
          printf("ERROR: failed to read json, super query result file not found\n");
4337
          goto PARSE_OVER;
4338
        }
4339 4340 4341
      }
    }
  }
4342

4343
  // super_table_query
4344 4345
  cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!superQuery) {
4346
    g_queryInfo.superQueryInfo.threadCnt = 1;
4347 4348
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
4349
    printf("ERROR: failed to read json, sub_table_query not found\n");
4350 4351
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
4352
  } else {
4353
    cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval");
4354
    if (subrate && subrate->type == cJSON_Number) {
4355
      g_queryInfo.superQueryInfo.queryInterval = subrate->valueint;
4356
    } else if (!subrate) {
4357
      g_queryInfo.superQueryInfo.queryInterval = 0;
4358 4359 4360 4361
    }

    cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
    if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
4362 4363
      if (superQueryTimes->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4364
          __func__, __LINE__, superQueryTimes->valueint);
4365 4366
        goto PARSE_OVER;
      }
4367 4368 4369 4370
      g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
    } else if (!superQueryTimes) {
      g_queryInfo.superQueryInfo.queryTimes = g_args.query_times;
    } else {
4371
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4372
          __func__, __LINE__);
4373
      goto PARSE_OVER;
4374
    }
4375

4376
    cJSON* threads = cJSON_GetObjectItem(superQuery, "threads");
4377
    if (threads && threads->type == cJSON_Number) {
4378 4379 4380 4381 4382 4383
      if (threads->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, threads input mistake\n",
          __func__, __LINE__);
        goto PARSE_OVER;

      }
4384
      g_queryInfo.superQueryInfo.threadCnt = threads->valueint;
4385
    } else if (!threads) {
4386
      g_queryInfo.superQueryInfo.threadCnt = 1;
4387
    }
4388

4389
    //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count");
4390
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
4391
    //  g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint;
4392
    //} else if (!subTblCnt) {
4393
    //  g_queryInfo.superQueryInfo.childTblCount = 0;
4394
    //}
4395

4396
    cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname");
4397 4398 4399
    if (stblname && stblname->type == cJSON_String
        && stblname->valuestring != NULL) {
      tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring,
4400
          MAX_TB_NAME_SIZE);
4401
    } else {
4402
      errorPrint("%s() LN%d, failed to read json, super table name input error\n",
4403
          __func__, __LINE__);
4404 4405
      goto PARSE_OVER;
    }
4406

4407 4408 4409 4410 4411 4412 4413
    cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode");
    if (superAsyncMode && superAsyncMode->type == cJSON_String
        && superAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE;
4414
      } else {
4415
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4416
            __func__, __LINE__);
4417 4418 4419
        goto PARSE_OVER;
      }
    } else {
4420
      g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
4421
    }
4422

4423 4424 4425 4426 4427 4428 4429 4430 4431
    cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval");
    if (superInterval && superInterval->type == cJSON_Number) {
      if (superInterval->valueint < 0) {
        errorPrint("%s() LN%d, failed to read json, interval input mistake\n",
            __func__, __LINE__);
        goto PARSE_OVER;
      }
      g_queryInfo.superQueryInfo.subscribeInterval = superInterval->valueint;
    } else if (!superInterval) {
4432 4433
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4434
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
4435
    }
4436

4437
    cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart");
4438 4439
    if (subrestart && subrestart->type == cJSON_String
        && subrestart->valuestring != NULL) {
4440
      if (0 == strcmp("yes", subrestart->valuestring)) {
4441
        g_queryInfo.superQueryInfo.subscribeRestart = 1;
4442
      } else if (0 == strcmp("no", subrestart->valuestring)) {
4443
        g_queryInfo.superQueryInfo.subscribeRestart = 0;
4444
      } else {
4445
        printf("ERROR: failed to read json, subscribe restart error\n");
4446 4447 4448
        goto PARSE_OVER;
      }
    } else {
4449
      g_queryInfo.superQueryInfo.subscribeRestart = 1;
4450
    }
4451

4452
    cJSON* subkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
4453
    if (subkeepProgress &&
4454 4455
            subkeepProgress->type == cJSON_String
            && subkeepProgress->valuestring != NULL) {
4456
      if (0 == strcmp("yes", subkeepProgress->valuestring)) {
4457
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
4458
      } else if (0 == strcmp("no", subkeepProgress->valuestring)) {
4459
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4460
      } else {
4461
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4462 4463 4464
        goto PARSE_OVER;
      }
    } else {
4465
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4466
    }
4467

4468
    // sqls
4469
    cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls");
4470
    if (!subsqls) {
4471
      g_queryInfo.superQueryInfo.sqlCount = 0;
4472
    } else if (subsqls->type != cJSON_Array) {
4473
      errorPrint("%s() LN%d: failed to read json, super sqls not found\n",
4474
          __func__, __LINE__);
4475
      goto PARSE_OVER;
4476
    } else {
4477 4478
      int superSqlSize = cJSON_GetArraySize(subsqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4479
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
4480
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4481 4482
        goto PARSE_OVER;
      }
4483

4484
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
4485
      for (int j = 0; j < superSqlSize; ++j) {
4486 4487
        cJSON* sql = cJSON_GetArrayItem(subsqls, j);
        if (sql == NULL) continue;
4488

4489
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
4490 4491 4492
        if (!sqlStr || sqlStr->type != cJSON_String
            || sqlStr->valuestring == NULL) {
          errorPrint("%s() LN%d, failed to read json, sql not found\n",
4493
              __func__, __LINE__);
4494 4495
          goto PARSE_OVER;
        }
4496
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring,
4497
            MAX_QUERY_SQL_LENGTH);
4498

4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510
        cJSON* superResubAfterConsume =
            cJSON_GetObjectItem(sql, "resubAfterConsume");
        if (superResubAfterConsume
                && superResubAfterConsume->type == cJSON_Number) {
            g_queryInfo.superQueryInfo.resubAfterConsume[j] =
                superResubAfterConsume->valueint;
        } else if (!superResubAfterConsume) {
            //printf("failed to read json, subscribe interval no found\n");
            //goto PARSE_OVER;
            g_queryInfo.superQueryInfo.resubAfterConsume[j] = 1;
        }

4511
        cJSON *result = cJSON_GetObjectItem(sql, "result");
4512 4513 4514
        if (result != NULL && result->type == cJSON_String
            && result->valuestring != NULL){
          tstrncpy(g_queryInfo.superQueryInfo.result[j],
4515
              result->valuestring, MAX_FILE_NAME_LEN);
4516
        } else if (NULL == result) {
4517
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4518
        }  else {
4519
          errorPrint("%s() LN%d, failed to read json, sub query result file not found\n",
4520
              __func__, __LINE__);
4521
          goto PARSE_OVER;
4522
        }
4523 4524
      }
    }
H
hzcheng 已提交
4525 4526
  }

4527
  ret = true;
H
hzcheng 已提交
4528

4529
PARSE_OVER:
4530 4531
  return ret;
}
H
hzcheng 已提交
4532

4533
static bool getInfoFromJsonFile(char* file) {
4534
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
4535

4536 4537 4538 4539 4540
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
4541

4542
  bool  ret = false;
4543
  int   maxLen = 6400000;
4544 4545 4546 4547 4548 4549 4550
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
4551
  }
H
Hui Li 已提交
4552

4553 4554 4555
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
4556
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
4557 4558
    goto PARSE_OVER;
  }
H
Hui Li 已提交
4559

4560 4561 4562
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
4563
      g_args.test_mode = INSERT_TEST;
4564
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
4565
      g_args.test_mode = QUERY_TEST;
4566
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
4567
      g_args.test_mode = SUBSCRIBE_TEST;
4568
    } else {
4569
      printf("ERROR: failed to read json, filetype not support\n");
4570 4571 4572
      goto PARSE_OVER;
    }
  } else if (!filetype) {
4573
    g_args.test_mode = INSERT_TEST;
4574
  } else {
4575
    printf("ERROR: failed to read json, filetype not found\n");
4576 4577
    goto PARSE_OVER;
  }
H
hzcheng 已提交
4578

4579
  if (INSERT_TEST == g_args.test_mode) {
4580
    ret = getMetaFromInsertJsonFile(root);
4581
  } else if ((QUERY_TEST == g_args.test_mode)
4582
          || (SUBSCRIBE_TEST == g_args.test_mode)) {
4583 4584
    ret = getMetaFromQueryJsonFile(root);
  } else {
4585
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n",
4586
            __func__, __LINE__);
4587
    goto PARSE_OVER;
4588
  }
4589

4590
PARSE_OVER:
4591 4592
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
4593
  fclose(fp);
4594 4595
  return ret;
}
H
hzcheng 已提交
4596

4597
static void prepareSampleData() {
4598
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4599 4600 4601
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
        (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]);
S
Shuaiqiang Chang 已提交
4602
      }
4603 4604 4605 4606
    }
  }
}

4607
static void postFreeResource() {
4608
  tmfclose(g_fpOfInsertResult);
4609
  for (int i = 0; i < g_Dbs.dbCount; i++) {
S
Shuduo Sang 已提交
4610
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
4611 4612 4613
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
4614
      }
4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4627
    }
4628 4629
  }
}
S
Shuaiqiang Chang 已提交
4630

4631 4632 4633
static int getRowDataFromSample(
        char* dataBuf, int64_t maxLen, int64_t timestamp,
      SSuperTable* superTblInfo, int64_t* sampleUsePos) {
4634
  if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
4635
/*    int ret = readSampleFromCsvFileToMem(superTblInfo);
4636
    if (0 != ret) {
4637
      tmfree(superTblInfo->sampleDataBuf);
4638
      superTblInfo->sampleDataBuf = NULL;
4639 4640
      return -1;
    }
4641
*/
4642 4643
    *sampleUsePos = 0;
  }
S
Shuaiqiang Chang 已提交
4644

4645
  int    dataLen = 0;
4646

4647
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
4648
          "(%" PRId64 ", ", timestamp);
4649
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
4650
          "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
4651
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
S
Shuaiqiang Chang 已提交
4652

4653
  (*sampleUsePos)++;
4654

4655 4656
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4657

4658 4659 4660 4661
static int64_t generateStbRowData(
        SSuperTable* stbInfo,
        char* recBuf, int64_t timestamp
        ) {
4662
  int64_t   dataLen = 0;
4663
  char  *pstr = recBuf;
4664
  int64_t maxLen = MAX_DATA_SIZE;
4665

4666 4667
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
          "(%" PRId64 ",", timestamp);
4668

4669
  for (int i = 0; i < stbInfo->columnCount; i++) {
4670 4671 4672 4673
    if ((0 == strncasecmp(stbInfo->columns[i].dataType,
                    "BINARY", strlen("BINARY")))
            || (0 == strncasecmp(stbInfo->columns[i].dataType,
                    "NCHAR", strlen("NCHAR")))) {
4674
      if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
4675
        errorPrint( "binary or nchar length overflow, max size:%u\n",
4676
                (uint32_t)TSDB_MAX_BINARY_LEN);
4677
        return -1;
4678
      }
4679

4680 4681
      char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
      if (NULL == buf) {
4682
        errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
4683
        return -1;
4684 4685
      }
      rand_string(buf, stbInfo->columns[i].dataLen);
4686
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf);
4687
      tmfree(buf);
4688
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4689
                "INT", 3)) {
4690
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4691
              "%d,", rand_int());
4692
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4693
                "BIGINT", 6)) {
4694
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4695
              "%"PRId64",", rand_bigint());
4696
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4697
                "FLOAT", 5)) {
4698
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4699
              "%f,", rand_float());
4700
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4701
                "DOUBLE", 6)) {
4702
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4703
              "%f,", rand_double());
4704
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4705
                "SMALLINT", 8)) {
4706
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4707
          "%d,", rand_smallint());
4708
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4709
          "TINYINT", strlen("TINYINT"))) {
4710
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4711
          "%d,", rand_tinyint());
4712
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4713
          "BOOL", strlen("BOOL"))) {
4714
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4715
          "%d,", rand_bool());
4716
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4717
          "TIMESTAMP", strlen("TIMESTAMP"))) {
4718
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4719
          "%"PRId64",", rand_bigint());
4720
    }  else {
4721
      errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType);
4722
      return -1;
4723
    }
S
Shuaiqiang Chang 已提交
4724
  }
4725

4726
  dataLen -= 1;
4727
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")");
4728

4729 4730 4731
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

  return strlen(recBuf);
4732
}
S
Shuaiqiang Chang 已提交
4733

4734
static int64_t generateData(char *recBuf, char **data_type,
4735
        int64_t timestamp, int lenOfBinary) {
4736 4737
  memset(recBuf, 0, MAX_DATA_SIZE);
  char *pstr = recBuf;
4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751
  pstr += sprintf(pstr, "(%" PRId64, timestamp);
  int c = 0;

  for (; c < MAX_NUM_DATATYPE; c++) {
    if (data_type[c] == NULL) {
      break;
    }
  }

  if (0 == c) {
    perror("data type error!");
    exit(-1);
  }

H
Haojun Liao 已提交
4752
  for (int i = 0; i < c; i++) {
4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763
    if (strcasecmp(data_type[i % c], "TINYINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_tinyint() );
    } else if (strcasecmp(data_type[i % c], "SMALLINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_smallint());
    } else if (strcasecmp(data_type[i % c], "INT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_int());
    } else if (strcasecmp(data_type[i % c], "BIGINT") == 0) {
      pstr += sprintf(pstr, ",%" PRId64, rand_bigint());
    } else if (strcasecmp(data_type[i % c], "FLOAT") == 0) {
      pstr += sprintf(pstr, ",%10.4f", rand_float());
    } else if (strcasecmp(data_type[i % c], "DOUBLE") == 0) {
4764
      double t = rand_double();
4765 4766
      pstr += sprintf(pstr, ",%20.8f", t);
    } else if (strcasecmp(data_type[i % c], "BOOL") == 0) {
4767
      bool b = rand_bool() & 1;
4768 4769
      pstr += sprintf(pstr, ",%s", b ? "true" : "false");
    } else if (strcasecmp(data_type[i % c], "BINARY") == 0) {
4770 4771
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4772
      pstr += sprintf(pstr, ",\"%s\"", s);
4773
      free(s);
4774
    } else if (strcasecmp(data_type[i % c], "NCHAR") == 0) {
4775 4776
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4777
      pstr += sprintf(pstr, ",\"%s\"", s);
4778 4779 4780
      free(s);
    }

4781
    if (strlen(recBuf) > MAX_DATA_SIZE) {
4782 4783 4784 4785 4786 4787 4788
      perror("column length too long, abort");
      exit(-1);
    }
  }

  pstr += sprintf(pstr, ")");

4789 4790
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

4791
  return (int32_t)strlen(recBuf);
4792 4793
}

4794
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
4795 4796
  char* sampleDataBuf = NULL;

4797
  sampleDataBuf = calloc(
4798
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
4799
  if (sampleDataBuf == NULL) {
4800
      errorPrint("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n",
4801 4802 4803 4804
              __func__, __LINE__,
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE,
              strerror(errno));
      return -1;
4805
  }
4806

4807 4808 4809 4810
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
4811 4812 4813 4814 4815
      errorPrint("%s() LN%d, read sample from csv file failed.\n",
          __func__, __LINE__);
      tmfree(sampleDataBuf);
      superTblInfo->sampleDataBuf = NULL;
      return -1;
H
Hui Li 已提交
4816
  }
4817

4818 4819 4820
  return 0;
}

S
Shuduo Sang 已提交
4821
static int64_t execInsert(threadInfo *pThreadInfo, uint64_t k)
4822 4823
{
  int affectedRows;
4824
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4825

4826
  verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
S
Shuduo Sang 已提交
4827
            __func__, __LINE__, pThreadInfo->buffer);
4828
  if (superTblInfo) {
4829
    if (superTblInfo->insertMode == TAOSC_IFACE) {
S
Shuduo Sang 已提交
4830 4831 4832
      affectedRows = queryDbExec(
              pThreadInfo->taos,
              pThreadInfo->buffer, INSERT_TYPE, false);
4833
    } else if (superTblInfo->insertMode == REST_IFACE) {
4834
      if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port,
S
Shuduo Sang 已提交
4835
                  pThreadInfo->buffer, NULL /* not set result file */)) {
4836
        affectedRows = -1;
4837
        printf("========restful return fail, threadID[%d]\n",
4838
            pThreadInfo->threadID);
4839 4840 4841
      } else {
        affectedRows = k;
      }
4842
    } else if (superTblInfo->insertMode == STMT_IFACE) {
S
Shuduo Sang 已提交
4843 4844 4845 4846 4847 4848
      debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt);
      if (0 != taos_stmt_execute(pThreadInfo->stmt)) {
        errorPrint("%s() LN%d, failied to execute insert statement\n",
                    __func__, __LINE__);
        exit(-1);
      }
S
Shuduo Sang 已提交
4849 4850

      affectedRows = k;
4851
    } else {
4852
      errorPrint("%s() LN%d: unknown insert mode: %d\n",
4853 4854
        __func__, __LINE__, superTblInfo->insertMode);
      affectedRows = 0;
4855
    }
4856
  } else {
S
Shuduo Sang 已提交
4857
    affectedRows = queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, INSERT_TYPE, false);
4858 4859 4860 4861 4862
  }

  return affectedRows;
}

4863
static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t tableSeq)
4864 4865
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4866
  if (superTblInfo) {
4867
    if (superTblInfo->childTblLimit > 0) {
4868 4869 4870
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName +
            (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
4871
    } else {
4872

4873
        verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n",
4874 4875 4876 4877 4878
                pThreadInfo->threadID, __func__, __LINE__,
                pThreadInfo->start_table_from,
                pThreadInfo->ntables, tableSeq);
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
4879 4880
    }
  } else {
4881
    snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"",
4882
        g_args.tb_prefix, tableSeq);
4883 4884 4885
  }
}

4886 4887 4888 4889 4890
static int64_t generateDataTailWithoutStb(
        uint64_t batch, char* buffer,
        int64_t remainderBufLen, int64_t insertRows,
        uint64_t startFrom, int64_t startTime,
        /* int64_t *pSamplePos, */int64_t *dataLen) {
4891

4892
  uint64_t len = 0;
4893 4894
  char *pstr = buffer;

4895
  verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch);
4896

4897
  int64_t k = 0;
4898
  for (k = 0; k < batch;) {
4899
    char data[MAX_DATA_SIZE];
4900 4901
    memset(data, 0, MAX_DATA_SIZE);

4902
    int64_t retLen = 0;
4903

4904 4905
    char **data_type = g_args.datatype;
    int lenOfBinary = g_args.len_of_binary;
4906

4907 4908 4909 4910 4911 4912 4913 4914 4915
    int64_t randTail = DEFAULT_TIMESTAMP_STEP * k;

    if (g_args.disorderRatio != 0) {
        int rand_num = taosRandom() % 100;
        if (rand_num < g_args.disorderRatio) {
          randTail = (randTail +
                  (taosRandom() % g_args.disorderRange + 1)) * (-1);

          debugPrint("rand data generated, back %"PRId64"\n", randTail);
4916
        }
4917 4918 4919
    } else {
        randTail = DEFAULT_TIMESTAMP_STEP * k;
    }
4920

4921 4922 4923
    retLen = generateData(data, data_type,
            startTime + randTail,
            lenOfBinary);
4924

4925
    if (len > remainderBufLen)
4926
        break;
4927

4928 4929 4930 4931 4932 4933 4934
    pstr += sprintf(pstr, "%s", data);
    k++;
    len += retLen;
    remainderBufLen -= retLen;

    verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n",
            __func__, __LINE__, len, k, buffer);
4935

4936
    startFrom ++;
4937

4938 4939 4940 4941
    if (startFrom >= insertRows) {
      break;
    }
  }
4942

4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983
  *dataLen = len;
  return k;
}

static int64_t generateStbDataTail(
        SSuperTable* superTblInfo,
        uint64_t batch, char* buffer,
        int64_t remainderBufLen, int64_t insertRows,
        uint64_t startFrom, int64_t startTime,
        int64_t *pSamplePos, int64_t *dataLen) {
  uint64_t len = 0;

  char *pstr = buffer;

  verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch);

  int64_t k = 0;
  for (k = 0; k < batch;) {
    char data[MAX_DATA_SIZE];
    memset(data, 0, MAX_DATA_SIZE);

    int64_t retLen = 0;

    if (0 == strncasecmp(superTblInfo->dataSource,
                "sample", strlen("sample"))) {
          retLen = getRowDataFromSample(
                  data,
                  remainderBufLen,
                  startTime + superTblInfo->timeStampStep * k,
                  superTblInfo,
                  pSamplePos);
    } else if (0 == strncasecmp(superTblInfo->dataSource,
                "rand", strlen("rand"))) {
        int64_t randTail = superTblInfo->timeStampStep * k;
        if (superTblInfo->disorderRatio > 0) {
            int rand_num = taosRandom() % 100;
            if(rand_num < superTblInfo->disorderRatio) {
                randTail = (randTail +
                        (taosRandom() % superTblInfo->disorderRange + 1)) * (-1);
                debugPrint("rand data generated, back %"PRId64"\n", randTail);
            }
4984
        }
4985

4986 4987 4988
        int64_t d = startTime + randTail;
        retLen = generateStbRowData(superTblInfo, data, d);
    }
4989

4990
    if (retLen > remainderBufLen) {
4991
        break;
4992 4993
    }

4994 4995 4996 4997 4998
    pstr += snprintf(pstr , retLen + 1, "%s", data);
    k++;
    len += retLen;
    remainderBufLen -= retLen;

4999
    verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n",
5000
            __func__, __LINE__, len, k, buffer);
5001 5002

    startFrom ++;
5003

5004
    if (startFrom >= insertRows) {
5005
      break;
5006
    }
5007
  }
5008

5009 5010 5011
  *dataLen = len;
  return k;
}
5012

5013 5014 5015

static int generateSQLHeadWithoutStb(char *tableName,
        char *dbName,
5016
        char *buffer, int remainderBufLen)
5017 5018
{
  int len;
5019 5020 5021

  char headBuf[HEAD_BUFF_LEN];

5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047
  len = snprintf(
          headBuf,
          HEAD_BUFF_LEN,
          "%s.%s values",
          dbName,
          tableName);

  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

  return len;
}

static int generateStbSQLHead(
        SSuperTable* superTblInfo,
        char *tableName, int32_t tableSeq,
        char *dbName,
        char *buffer, int remainderBufLen)
{
  int len;

  char headBuf[HEAD_BUFF_LEN];

  if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
5048 5049
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
5050
            tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq);
5051
      } else {
5052 5053 5054
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
5055 5056
      }
      if (NULL == tagsValBuf) {
5057
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n",
5058
            __func__, __LINE__);
5059 5060 5061
        return -1;
      }

5062 5063
      len = snprintf(
          headBuf,
5064 5065
                  HEAD_BUFF_LEN,
                  "%s.%s using %s.%s tags %s values",
5066
                  dbName,
5067
                  tableName,
5068
                  dbName,
5069 5070
                  superTblInfo->sTblName,
                  tagsValBuf);
5071 5072
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
5073 5074
      len = snprintf(
          headBuf,
5075 5076
                  HEAD_BUFF_LEN,
                  "%s.%s values",
5077
                  dbName,
5078
                  tableName);
5079
    } else {
5080 5081
      len = snprintf(
          headBuf,
5082 5083
                  HEAD_BUFF_LEN,
                  "%s.%s values",
5084
                  dbName,
5085
                  tableName);
5086 5087
  }

5088 5089 5090 5091 5092
  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

5093 5094 5095
  return len;
}

5096
static int64_t generateInterlaceDataBuffer(
5097 5098
        char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes,
        uint64_t tableSeq,
5099
        threadInfo *pThreadInfo, char *buffer,
5100
        int64_t insertRows,
5101
        int64_t startTime,
5102
        uint64_t *pRemainderBufLen)
5103
{
5104
  assert(buffer);
5105 5106 5107
  char *pstr = buffer;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

5108 5109 5110 5111
  int headLen = generateStbSQLHead(
          superTblInfo,
          tableName, tableSeq, pThreadInfo->db_name,
            pstr, *pRemainderBufLen);
5112 5113 5114 5115 5116

  if (headLen <= 0) {
    return 0;
  }
  // generate data buffer
5117
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n",
5118
            pThreadInfo->threadID, __func__, __LINE__, i, buffer);
5119 5120 5121 5122

  pstr += headLen;
  *pRemainderBufLen -= headLen;

5123
  int64_t dataLen = 0;
5124

5125
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%"PRIu64" batchPerTbl = %"PRIu64"\n",
5126 5127
            pThreadInfo->threadID, __func__, __LINE__,
            i, batchPerTblTimes, batchPerTbl);
5128

5129
  int64_t k;
5130 5131 5132 5133
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
      startTime = taosGetTimestamp(pThreadInfo->time_precision);
    }
5134 5135 5136 5137 5138 5139

    k = generateStbDataTail(
            superTblInfo,
            batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
            startTime,
            &(pThreadInfo->samplePos), &dataLen);
5140
  } else {
5141 5142 5143 5144 5145
    startTime = 1500000000000;
    k = generateDataTailWithoutStb(
            batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
            startTime,
            /* &(pThreadInfo->samplePos), */&dataLen);
5146
  }
5147 5148

  if (k == batchPerTbl) {
5149 5150 5151
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
  } else {
5152
    debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n",
5153
            __func__, __LINE__, k, batchPerTbl);
5154 5155
    pstr -= headLen;
    pstr[0] = '\0';
5156
    k = 0;
5157 5158 5159 5160 5161
  }

  return k;
}

5162
static int64_t generateProgressiveDataBuffer(
5163
        char *tableName,
5164
        int64_t tableSeq,
5165 5166
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
5167
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos,
5168
        int64_t *pRemainderBufLen)
5169 5170 5171 5172
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  assert(buffer != NULL);
5173
  char *pstr = buffer;
5174

5175
  memset(buffer, 0, *pRemainderBufLen);
5176

5177 5178 5179 5180 5181 5182 5183 5184 5185 5186
  int64_t headLen;

  if (superTblInfo) {
      headLen = generateStbSQLHead(
              superTblInfo,
          tableName, tableSeq, pThreadInfo->db_name,
          buffer, *pRemainderBufLen);
  } else {
      headLen = generateSQLHeadWithoutStb(
          tableName, pThreadInfo->db_name,
5187
          buffer, *pRemainderBufLen);
5188
  }
5189 5190 5191 5192

  if (headLen <= 0) {
    return 0;
  }
5193
  pstr += headLen;
5194
  *pRemainderBufLen -= headLen;
5195

5196
  int64_t dataLen;
5197 5198 5199 5200 5201 5202
  int64_t k;

  if (superTblInfo) {
    k = generateStbDataTail(superTblInfo,
          g_args.num_of_RPR, pstr, *pRemainderBufLen,
          insertRows, startFrom,
5203 5204
          startTime,
          pSamplePos, &dataLen);
5205 5206 5207 5208 5209 5210
  } else {
    k = generateDataTailWithoutStb(
          g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom,
          startTime,
          /*pSamplePos, */&dataLen);
  }
5211

5212 5213
  return k;
}
5214

5215 5216 5217 5218 5219 5220 5221 5222 5223
static void printStatPerThread(threadInfo *pThreadInfo)
{
  fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n",
          pThreadInfo->threadID,
          pThreadInfo->totalInsertRows,
          pThreadInfo->totalAffectedRows,
          (double)(pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay/1000.0)));
}

5224
// sync write interlace data
5225
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
5226
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
5227 5228
         pThreadInfo->threadID, __func__, __LINE__);

5229
  int64_t insertRows;
5230
  uint64_t interlaceRows;
5231 5232

  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5233

5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246
  if (superTblInfo) {
    insertRows = superTblInfo->insertRows;

    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    insertRows = g_args.num_of_DPT;
    interlaceRows = g_args.interlace_rows;
  }
5247

5248
  if (interlaceRows > insertRows)
H
Haojun Liao 已提交
5249
    interlaceRows = insertRows;
5250

5251 5252
  if (interlaceRows > g_args.num_of_RPR)
    interlaceRows = g_args.num_of_RPR;
5253

5254
  int progOrInterlace;
5255 5256

  if (interlaceRows > 0) {
5257
    progOrInterlace= INTERLACE_INSERT_MODE;
5258
  } else {
5259
    progOrInterlace = PROGRESSIVE_INSERT_MODE;
5260 5261
  }

5262
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
S
Shuduo Sang 已提交
5263 5264
  pThreadInfo->buffer = calloc(maxSqlLen, 1);
  if (NULL == pThreadInfo->buffer) {
5265
    errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5266
              __func__, __LINE__, maxSqlLen, strerror(errno));
5267 5268 5269
    return NULL;
  }

5270
  char tableName[TSDB_TABLE_NAME_LEN];
5271 5272 5273 5274

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

5275 5276
  int64_t nTimeStampStep =
      superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
5277

5278
  uint64_t insert_interval =
5279
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5280 5281
  uint64_t st = 0;
  uint64_t et = UINT64_MAX;
5282

5283 5284 5285
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5286

5287
  uint64_t tableSeq = pThreadInfo->start_table_from;
5288

S
Shuduo Sang 已提交
5289
  debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n",
5290 5291
          pThreadInfo->threadID, __func__, __LINE__,
          pThreadInfo->start_table_from,
5292
          pThreadInfo->ntables, insertRows);
5293 5294 5295

  int64_t startTime = pThreadInfo->start_time;

5296 5297
  uint64_t batchPerTbl = interlaceRows;
  uint64_t batchPerTblTimes;
5298

5299
  if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) {
5300
    batchPerTblTimes =
5301
        g_args.num_of_RPR / interlaceRows;
5302 5303 5304 5305
  } else {
    batchPerTblTimes = 1;
  }

5306
  uint64_t generatedRecPerTbl = 0;
5307
  bool flagSleep = true;
5308
  uint64_t sleepTimeTotal = 0;
5309

5310 5311 5312
  char *strInsertInto = "insert into ";
  int nInsertBufLen = strlen(strInsertInto);

5313
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
5314
    if ((flagSleep) && (insert_interval)) {
5315 5316
        st = taosGetTimestampMs();
        flagSleep = false;
5317 5318
    }
    // generate data
S
Shuduo Sang 已提交
5319
    memset(pThreadInfo->buffer, 0, maxSqlLen);
5320
    uint64_t remainderBufLen = maxSqlLen;
5321

S
Shuduo Sang 已提交
5322
    char *pstr = pThreadInfo->buffer;
5323 5324 5325 5326 5327

    int len = snprintf(pstr, nInsertBufLen + 1, "%s", strInsertInto);
    pstr += len;
    remainderBufLen -= len;

5328
    uint64_t recOfBatch = 0;
5329

5330
    for (uint64_t i = 0; i < batchPerTblTimes; i ++) {
5331
      getTableName(tableName, pThreadInfo, tableSeq);
5332 5333
      if (0 == strlen(tableName)) {
        errorPrint("[%d] %s() LN%d, getTableName return null\n",
5334
            pThreadInfo->threadID, __func__, __LINE__);
S
Shuduo Sang 已提交
5335
        free(pThreadInfo->buffer);
5336 5337
        return NULL;
      }
5338

5339
      uint64_t oldRemainderLen = remainderBufLen;
5340
      int64_t generated = generateInterlaceDataBuffer(
5341 5342 5343 5344 5345 5346
        tableName, batchPerTbl, i, batchPerTblTimes,
        tableSeq,
        pThreadInfo, pstr,
        insertRows,
        startTime,
        &remainderBufLen);
5347

5348 5349
      debugPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5350
      if (generated < 0) {
5351
        errorPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
5352
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5353
        goto free_of_interlace;
5354 5355
      } else if (generated == 0) {
        break;
5356 5357
      }

5358
      tableSeq ++;
5359
      recOfBatch += batchPerTbl;
5360
      pstr += (oldRemainderLen - remainderBufLen);
5361
//      startTime += batchPerTbl * superTblInfo->timeStampStep;
5362
      pThreadInfo->totalInsertRows += batchPerTbl;
5363
      verbosePrint("[%d] %s() LN%d batchPerTbl=%"PRId64" recOfBatch=%"PRId64"\n",
5364 5365
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
5366

5367
      if (progOrInterlace == INTERLACE_INSERT_MODE) {
5368 5369 5370 5371
          if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
            // turn to first table
            tableSeq = pThreadInfo->start_table_from;
            generatedRecPerTbl += batchPerTbl;
5372

5373 5374
            startTime = pThreadInfo->start_time
              + generatedRecPerTbl * nTimeStampStep;
5375

5376 5377 5378
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;
5379

5380 5381 5382
            int remainRows = insertRows - generatedRecPerTbl;
            if ((remainRows > 0) && (batchPerTbl > remainRows))
              batchPerTbl = remainRows;
5383

5384 5385 5386 5387
            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
          }
      }
5388

5389
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n",
5390 5391
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);
5392 5393

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
5394 5395 5396
        break;
    }

5397
    verbosePrint("[%d] %s() LN%d recOfBatch=%"PRIu64" totalInsertRows=%"PRIu64"\n",
5398 5399
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
5400
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
S
Shuduo Sang 已提交
5401
           pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer);
5402

5403
    startTs = taosGetTimestampMs();
5404

5405 5406 5407 5408 5409 5410 5411
    if (recOfBatch == 0) {
      errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n",
              pThreadInfo->threadID, __func__, __LINE__,
              recOfBatch);
      errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n");
      goto free_of_interlace;
    }
S
Shuduo Sang 已提交
5412
    int64_t affectedRows = execInsert(pThreadInfo, recOfBatch);
5413

5414
    endTs = taosGetTimestampMs();
5415 5416
    uint64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n",
5417
            __func__, __LINE__, delay);
5418 5419 5420
    verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5421 5422 5423 5424 5425 5426

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

5427 5428
    if (recOfBatch != affectedRows) {
        errorPrint("[%d] %s() LN%d execInsert insert %"PRIu64", affected rows: %"PRId64"\n%s\n",
5429
                pThreadInfo->threadID, __func__, __LINE__,
S
Shuduo Sang 已提交
5430
                recOfBatch, affectedRows, pThreadInfo->buffer);
5431
        goto free_of_interlace;
5432
    }
5433

5434
    pThreadInfo->totalAffectedRows += affectedRows;
5435

5436 5437
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
5438
      printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n",
5439 5440 5441
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5442 5443
      lastPrintTime = currentPrintTime;
    }
5444

5445
    if ((insert_interval) && flagSleep) {
5446
      et = taosGetTimestampMs();
5447

5448
      if (insert_interval > (et - st) ) {
S
Shuduo Sang 已提交
5449 5450
        uint64_t sleepTime = insert_interval - (et -st);
        performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n",
5451
                    __func__, __LINE__, sleepTime);
5452 5453 5454
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
5455
    }
5456 5457
  }

5458
free_of_interlace:
S
Shuduo Sang 已提交
5459
  tmfree(pThreadInfo->buffer);
5460
  printStatPerThread(pThreadInfo);
5461 5462 5463
  return NULL;
}

5464
// sync insertion progressive data
5465
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
5466
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
5467

5468
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5469
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
5470 5471 5472 5473 5474 5475
  int64_t timeStampStep =
      superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
  int64_t insertRows =
        (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
  verbosePrint("%s() LN%d insertRows=%"PRId64"\n",
            __func__, __LINE__, insertRows);
5476

S
Shuduo Sang 已提交
5477 5478
  pThreadInfo->buffer = calloc(maxSqlLen, 1);
  if (NULL == pThreadInfo->buffer) {
5479
    errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5480 5481
              maxSqlLen,
              strerror(errno));
5482 5483
    return NULL;
  }
5484

5485 5486 5487
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5488

5489
/*  int insert_interval =
5490
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5491 5492
  uint64_t st = 0;
  uint64_t et = 0xffffffff;
5493
  */
5494

5495 5496
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
5497

5498
  pThreadInfo->samplePos = 0;
5499

5500 5501 5502
  for (uint64_t tableSeq = pThreadInfo->start_table_from;
          tableSeq <= pThreadInfo->end_table_to;
          tableSeq ++) {
5503
    int64_t start_time = pThreadInfo->start_time;
5504

5505
    for (uint64_t i = 0; i < insertRows;) {
5506 5507
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
5508
      verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n",
5509 5510
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);
5511

5512
      int64_t remainderBufLen = maxSqlLen;
S
Shuduo Sang 已提交
5513
      char *pstr = pThreadInfo->buffer;
5514 5515 5516 5517 5518 5519 5520
      int nInsertBufLen = strlen("insert into ");

      int len = snprintf(pstr, nInsertBufLen + 1, "%s", "insert into ");

      pstr += len;
      remainderBufLen -= len;

5521
      int64_t generated = generateProgressiveDataBuffer(
5522 5523 5524 5525
              tableName, tableSeq, pThreadInfo, pstr, insertRows,
            i, start_time,
            &(pThreadInfo->samplePos),
            &remainderBufLen);
5526 5527 5528
      if (generated > 0)
        i += generated;
      else
5529
        goto free_of_progressive;
5530

5531
      start_time +=  generated * timeStampStep;
5532
      pThreadInfo->totalInsertRows += generated;
5533

5534
      startTs = taosGetTimestampMs();
5535

S
Shuduo Sang 已提交
5536
      int64_t affectedRows = execInsert(pThreadInfo, generated);
5537

5538
      endTs = taosGetTimestampMs();
5539
      uint64_t delay = endTs - startTs;
5540
      performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n",
5541
              __func__, __LINE__, delay);
5542 5543 5544
      verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5545

5546 5547 5548 5549
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
5550

5551 5552 5553 5554 5555
      if (affectedRows < 0) {
        errorPrint("%s() LN%d, affected rows: %"PRId64"\n",
                __func__, __LINE__, affectedRows);
        goto free_of_progressive;
      }
5556 5557 5558

      pThreadInfo->totalAffectedRows += affectedRows;

5559 5560
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
5561
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
5562 5563 5564
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5565 5566 5567
        lastPrintTime = currentPrintTime;
      }

5568
      if (i >= insertRows)
5569 5570 5571
        break;
    }   // num_of_DPT

5572 5573
    if ((g_args.verbose_print) &&
      (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) &&
5574 5575
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
5576
          verbosePrint("%s() LN%d samplePos=%"PRId64"\n",
5577
                  __func__, __LINE__, pThreadInfo->samplePos);
5578
    }
5579
  } // tableSeq
5580

5581
free_of_progressive:
S
Shuduo Sang 已提交
5582
  tmfree(pThreadInfo->buffer);
5583
  printStatPerThread(pThreadInfo);
5584
  return NULL;
H
Hui Li 已提交
5585 5586
}

5587 5588
static void* syncWrite(void *sarg) {

5589 5590
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5591

5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603
  int interlaceRows;

  if (superTblInfo) {
    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    interlaceRows = g_args.interlace_rows;
  }
5604

5605
  if (interlaceRows > 0) {
5606
    // interlace mode
5607
    return syncWriteInterlace(pThreadInfo);
5608 5609
  } else {
    // progressive mode
5610
    return syncWriteProgressive(pThreadInfo);
5611 5612 5613
  }
}

5614
static void callBack(void *param, TAOS_RES *res, int code) {
5615 5616
  threadInfo* pThreadInfo = (threadInfo*)param;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5617

5618 5619
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5620
  if (insert_interval) {
5621 5622 5623
    pThreadInfo->et = taosGetTimestampMs();
    if ((pThreadInfo->et - pThreadInfo->st) < insert_interval) {
      taosMsleep(insert_interval - (pThreadInfo->et - pThreadInfo->st)); // ms
5624
    }
H
Hui Li 已提交
5625
  }
5626

5627
  char *buffer = calloc(1, pThreadInfo->superTblInfo->maxSqlLen);
5628
  char data[MAX_DATA_SIZE];
5629
  char *pstr = buffer;
5630 5631
  pstr += sprintf(pstr, "insert into %s.%s%"PRId64" values",
          pThreadInfo->db_name, pThreadInfo->tb_prefix,
5632
          pThreadInfo->start_table_from);
5633 5634 5635 5636 5637 5638 5639
//  if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
  if (pThreadInfo->counter >= g_args.num_of_RPR) {
    pThreadInfo->start_table_from++;
    pThreadInfo->counter = 0;
  }
  if (pThreadInfo->start_table_from > pThreadInfo->end_table_to) {
    tsem_post(&pThreadInfo->lock_sem);
5640 5641 5642 5643
    free(buffer);
    taos_free_result(res);
    return;
  }
5644

5645
  for (int i = 0; i < g_args.num_of_RPR; i++) {
5646
    int rand_num = taosRandom() % 100;
5647
    if (0 != pThreadInfo->superTblInfo->disorderRatio
5648
            && rand_num < pThreadInfo->superTblInfo->disorderRatio) {
5649 5650 5651
      int64_t d = pThreadInfo->lastTs
          - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1);
      generateStbRowData(pThreadInfo->superTblInfo, data, d);
5652
    } else {
5653 5654
      generateStbRowData(pThreadInfo->superTblInfo,
              data, pThreadInfo->lastTs += 1000);
H
Hui Li 已提交
5655
    }
5656
    pstr += sprintf(pstr, "%s", data);
5657
    pThreadInfo->counter++;
H
Hui Li 已提交
5658

5659
    if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
5660
      break;
H
Hui Li 已提交
5661 5662
    }
  }
5663

5664
  if (insert_interval) {
5665
    pThreadInfo->st = taosGetTimestampMs();
5666
  }
5667
  taos_query_a(pThreadInfo->taos, buffer, callBack, pThreadInfo);
5668
  free(buffer);
H
Hui Li 已提交
5669

5670
  taos_free_result(res);
H
Hui Li 已提交
5671 5672
}

5673
static void *asyncWrite(void *sarg) {
5674 5675
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5676

5677 5678 5679
  pThreadInfo->st = 0;
  pThreadInfo->et = 0;
  pThreadInfo->lastTs = pThreadInfo->start_time;
5680

5681
  int insert_interval =
5682
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5683
  if (insert_interval) {
5684
    pThreadInfo->st = taosGetTimestampMs();
H
Hui Li 已提交
5685
  }
5686
  taos_query_a(pThreadInfo->taos, "show databases", callBack, pThreadInfo);
5687

5688
  tsem_wait(&(pThreadInfo->lock_sem));
H
Hui Li 已提交
5689 5690 5691 5692

  return NULL;
}

5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718
static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in *serv_addr)
{
  uint16_t rest_port = port + TSDB_PORT_HTTP;
  struct hostent *server = gethostbyname(host);
  if ((server == NULL) || (server->h_addr == NULL)) {
    errorPrint("%s", "ERROR, no such host");
    return -1;
  }

  debugPrint("h_name: %s\nh_addr=%p\nh_addretype: %s\nh_length: %d\n",
            server->h_name,
            server->h_addr,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

  memset(serv_addr, 0, sizeof(struct sockaddr_in));
  serv_addr->sin_family = AF_INET;
  serv_addr->sin_port = htons(rest_port);
#ifdef WINDOWS
  serv_addr->sin_addr.s_addr = inet_addr(host);
#else
  memcpy(&(serv_addr->sin_addr.s_addr), server->h_addr, server->h_length);
#endif
  return 0;
}

5719
static void startMultiThreadInsertData(int threads, char* db_name,
5720
        char* precision,SSuperTable* superTblInfo) {
5721

5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737
  //TAOS* taos;
  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
  //  if (NULL == taos) {
  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
  //    exit(-1);
  //  }
  //}

  int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
  if (0 != precision[0]) {
    if (0 == strncasecmp(precision, "ms", 2)) {
      timePrec = TSDB_TIME_PRECISION_MILLI;
    }  else if (0 == strncasecmp(precision, "us", 2)) {
      timePrec = TSDB_TIME_PRECISION_MICRO;
    }  else {
5738
      errorPrint("Not support precision: %s\n", precision);
5739 5740 5741 5742
      exit(-1);
    }
  }

5743
  int64_t start_time;
5744 5745
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
5746
        start_time = taosGetTimestamp(timePrec);
5747
    } else {
5748
      if (TSDB_CODE_SUCCESS != taosParseTime(
5749 5750 5751 5752 5753
        superTblInfo->startTimestamp,
        &start_time,
        strlen(superTblInfo->startTimestamp),
        timePrec, 0)) {
          ERROR_EXIT("failed to parse time!\n");
5754
      }
5755
    }
5756
  } else {
5757
     start_time = 1500000000000;
5758 5759
  }

5760
  int64_t start = taosGetTimestampMs();
5761

5762
  // read sample data from file first
5763
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5764
              "sample", strlen("sample")))) {
5765
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5766
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
5767
              __func__, __LINE__);
5768 5769 5770 5771
      exit(-1);
    }
  }

5772
  // read sample data from file first
5773
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5774
              "sample", strlen("sample")))) {
5775
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5776
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
5777
              __func__, __LINE__);
5778 5779 5780 5781
      exit(-1);
    }
  }

5782
  TAOS* taos0 = taos_connect(
5783 5784
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
5785
  if (NULL == taos0) {
5786
    errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
5787
                __func__, __LINE__, taos_errstr(NULL));
5788 5789 5790
    exit(-1);
  }

S
Shuduo Sang 已提交
5791
  int64_t ntables = 0;
S
Shuduo Sang 已提交
5792
  uint64_t startFrom;
5793

5794
  if (superTblInfo) {
5795 5796
    int64_t limit;
    uint64_t offset;
5797

H
Haojun Liao 已提交
5798
    if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) &&
5799
            ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) {
H
Haojun Liao 已提交
5800
      printf("WARNING: offset and limit will not be used since the child tables not exists!\n");
5801 5802
    }

5803
    if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) {
5804 5805
      if ((superTblInfo->childTblLimit < 0)
          || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit)
5806
            > (superTblInfo->childTblCount))) {
5807 5808 5809 5810 5811 5812
        superTblInfo->childTblLimit =
            superTblInfo->childTblCount - superTblInfo->childTblOffset;
      }

      offset = superTblInfo->childTblOffset;
      limit = superTblInfo->childTblLimit;
5813
    } else {
5814 5815
      limit = superTblInfo->childTblCount;
      offset = 0;
5816 5817
    }

5818 5819 5820
    ntables = limit;
    startFrom = offset;

5821 5822 5823 5824
    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
        && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
            > superTblInfo->childTblCount)) {
      printf("WARNING: specified offset + limit > child table count!\n");
5825
      prompt();
5826 5827 5828
    }

    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
5829
            && (0 == superTblInfo->childTblLimit)) {
5830
      printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
5831
      prompt();
5832 5833
    }

5834
    superTblInfo->childTblName = (char*)calloc(1,
5835
        limit * TSDB_TABLE_NAME_LEN);
5836
    if (superTblInfo->childTblName == NULL) {
5837
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
5838
      taos_close(taos0);
5839 5840 5841
      exit(-1);
    }

5842
    int64_t childTblCount;
5843
    getChildNameOfSuperTableWithLimitAndOffset(
5844
        taos0,
5845 5846 5847 5848
        db_name, superTblInfo->sTblName,
        &superTblInfo->childTblName, &childTblCount,
        limit,
        offset);
5849 5850 5851
  } else {
    ntables = g_args.num_of_tables;
    startFrom = 0;
5852
  }
5853

5854
  taos_close(taos0);
5855

S
Shuduo Sang 已提交
5856
  int64_t a = ntables / threads;
5857 5858 5859 5860 5861
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

S
Shuduo Sang 已提交
5862
  int64_t b = 0;
5863 5864 5865 5866
  if (threads != 0) {
    b = ntables % threads;
  }

5867
  if ((superTblInfo)
5868 5869 5870 5871
      && (superTblInfo->insertMode == REST_IFACE)) {
      if (convertHostToServAddr(g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) {
        exit(-1);
      }
5872 5873
  }

5874 5875 5876 5877 5878 5879 5880 5881 5882
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  assert(pids != NULL);

  threadInfo *infos = calloc(1, threads * sizeof(threadInfo));
  assert(infos != NULL);

  memset(pids, 0, threads * sizeof(pthread_t));
  memset(infos, 0, threads * sizeof(threadInfo));

H
Hui Li 已提交
5883
  for (int i = 0; i < threads; i++) {
5884
    threadInfo *t_info = infos + i;
H
Hui Li 已提交
5885 5886
    t_info->threadID = i;
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
5887
    t_info->time_precision = timePrec;
5888 5889 5890
    t_info->superTblInfo = superTblInfo;

    t_info->start_time = start_time;
5891
    t_info->minDelay = UINT64_MAX;
5892

5893
    if ((NULL == superTblInfo) ||
5894
            (superTblInfo->insertMode != REST_IFACE)) {
5895
      //t_info->taos = taos;
5896
      t_info->taos = taos_connect(
5897 5898
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
5899
      if (NULL == t_info->taos) {
5900
        errorPrint(
5901 5902
                "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n",
                __func__, __LINE__,
5903
                taos_errstr(NULL));
5904
        free(infos);
5905 5906
        exit(-1);
      }
5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919

      if ((superTblInfo) && (superTblInfo->insertMode == STMT_IFACE)) {
        t_info->stmt = taos_stmt_init(t_info->taos);
        if (NULL == t_info->stmt) {
            errorPrint(
                "%s() LN%d, failed init stmt, reason: %s\n",
                __func__, __LINE__,
                taos_errstr(NULL));
            free(pids);
            free(infos);
            exit(-1);
        }
      }
5920 5921 5922 5923
    } else {
      t_info->taos = NULL;
    }

5924
/*    if ((NULL == superTblInfo)
5925
            || (0 == superTblInfo->multiThreadWriteOneTbl)) {
5926
            */
5927 5928 5929 5930
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
5931
/*    } else {
5932 5933
      t_info->start_table_from = 0;
      t_info->ntables = superTblInfo->childTblCount;
5934 5935
      t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint();
    }
5936
*/
5937
    tsem_init(&(t_info->lock_sem), 0, 0);
5938
    if (ASYNC_MODE == g_Dbs.asyncMode) {
5939
      pthread_create(pids + i, NULL, asyncWrite, t_info);
5940 5941
    } else {
      pthread_create(pids + i, NULL, syncWrite, t_info);
5942
    }
H
Hui Li 已提交
5943
  }
5944

H
Hui Li 已提交
5945 5946 5947 5948
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

5949 5950 5951 5952
  uint64_t totalDelay = 0;
  uint64_t maxDelay = 0;
  uint64_t minDelay = UINT64_MAX;
  uint64_t cntDelay = 1;
5953 5954
  double  avgDelay = 0;

H
Hui Li 已提交
5955
  for (int i = 0; i < threads; i++) {
5956 5957
    threadInfo *t_info = infos + i;

S
TD-1057  
Shengliang Guan 已提交
5958
    tsem_destroy(&(t_info->lock_sem));
5959 5960 5961 5962

    if (t_info->stmt) {
      taos_stmt_close(t_info->stmt);
    }
5963 5964
    taos_close(t_info->taos);

5965
    debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n",
5966 5967 5968
            __func__, __LINE__,
            t_info->threadID, t_info->totalInsertRows,
            t_info->totalAffectedRows);
5969
    if (superTblInfo) {
5970 5971
        superTblInfo->totalAffectedRows += t_info->totalAffectedRows;
        superTblInfo->totalInsertRows += t_info->totalInsertRows;
5972
    } else {
5973 5974
        g_args.totalAffectedRows += t_info->totalAffectedRows;
        g_args.totalInsertRows += t_info->totalInsertRows;
5975
    }
5976 5977 5978 5979

    totalDelay  += t_info->totalDelay;
    cntDelay   += t_info->cntDelay;
    if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay;
5980
    if (t_info->minDelay < minDelay) minDelay = t_info->minDelay;
H
Hui Li 已提交
5981
  }
5982
  cntDelay -= 1;
H
Hui Li 已提交
5983

5984
  if (cntDelay == 0)    cntDelay = 1;
5985 5986
  avgDelay = (double)totalDelay / cntDelay;

5987 5988
  int64_t end = taosGetTimestampMs();
  int64_t t = end - start;
5989 5990

  if (superTblInfo) {
5991
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
5992 5993 5994 5995
          t / 1000.0, superTblInfo->totalInsertRows,
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
5996 5997 5998 5999

    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
6000 6001 6002 6003
          t / 1000.0, superTblInfo->totalInsertRows,
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
6004
    }
6005
  } else {
6006
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
6007 6008 6009 6010
          t / 1000.0, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          (double)g_args.totalInsertRows / (t / 1000.0));
6011 6012 6013
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
6014 6015 6016 6017
          t * 1000.0, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          (double)g_args.totalInsertRows / (t / 1000.0));
6018
    }
6019
  }
6020

6021
  fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
6022
          avgDelay, maxDelay, minDelay);
6023 6024
  if (g_fpOfInsertResult) {
    fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
6025
          avgDelay, maxDelay, minDelay);
6026
  }
6027

6028 6029
  //taos_close(taos);

H
Hui Li 已提交
6030
  free(pids);
6031
  free(infos);
H
Hui Li 已提交
6032 6033
}

6034
static void *readTable(void *sarg) {
6035
#if 1
6036
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
6037 6038
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
6039
  uint64_t sTime = rinfo->start_time;
H
hzcheng 已提交
6040 6041
  char *tb_prefix = rinfo->tb_prefix;
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
6042
  if (NULL == fp) {
6043
    errorPrint( "fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
H
Hui Li 已提交
6044 6045
    return NULL;
  }
6046

6047
  int64_t num_of_DPT;
6048
/*  if (rinfo->superTblInfo) {
6049 6050
    num_of_DPT = rinfo->superTblInfo->insertRows; //  nrecords_per_table;
  } else {
6051
  */
6052
      num_of_DPT = g_args.num_of_DPT;
6053
//  }
6054

S
Shuduo Sang 已提交
6055 6056
  int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
  int64_t totalData = num_of_DPT * num_of_tables;
6057
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6058 6059 6060 6061 6062

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
S
Shuduo Sang 已提交
6063
  printf("%"PRId64" records:\n", totalData);
H
hzcheng 已提交
6064 6065
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

6066
  for (int j = 0; j < n; j++) {
H
hzcheng 已提交
6067
    double totalT = 0;
6068
    uint64_t count = 0;
6069 6070
    for (int64_t i = 0; i < num_of_tables; i++) {
      sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64,
6071
              aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
6072

6073
      double t = taosGetTimestampMs();
S
Shuaiqiang Chang 已提交
6074 6075
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
6076

S
Shuaiqiang Chang 已提交
6077
      if (code != 0) {
6078
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6079
        taos_free_result(pSql);
H
hzcheng 已提交
6080
        taos_close(taos);
6081
        fclose(fp);
6082
        return NULL;
H
hzcheng 已提交
6083 6084
      }

6085
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6086 6087 6088
        count++;
      }

6089
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6090 6091
      totalT += t;

S
Shuaiqiang Chang 已提交
6092
      taos_free_result(pSql);
H
hzcheng 已提交
6093 6094
    }

6095
    fprintf(fp, "|%10s  |   %"PRId64"   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
6096
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
6097
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
6098
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000);
H
hzcheng 已提交
6099 6100 6101
  }
  fprintf(fp, "\n");
  fclose(fp);
6102
#endif
H
hzcheng 已提交
6103 6104 6105
  return NULL;
}

6106
static void *readMetric(void *sarg) {
6107
#if 1
6108
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
6109 6110 6111
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
6112 6113 6114 6115
  if (NULL == fp) {
    printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
    return NULL;
  }
6116

6117
  int64_t num_of_DPT = rinfo->superTblInfo->insertRows;
S
Shuduo Sang 已提交
6118 6119
  int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
  int64_t totalData = num_of_DPT * num_of_tables;
6120
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6121 6122 6123 6124 6125

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
S
Shuduo Sang 已提交
6126 6127
  printf("%"PRId64" records:\n", totalData);
  fprintf(fp, "Querying On %"PRId64" records:\n", totalData);
H
hzcheng 已提交
6128 6129

  for (int j = 0; j < n; j++) {
6130
    char condition[COND_BUF_LEN] = "\0";
B
Bomin Zhang 已提交
6131
    char tempS[64] = "\0";
H
hzcheng 已提交
6132

6133
    int64_t m = 10 < num_of_tables ? 10 : num_of_tables;
H
hzcheng 已提交
6134

6135
    for (int64_t i = 1; i <= m; i++) {
H
hzcheng 已提交
6136
      if (i == 1) {
6137
        sprintf(tempS, "t1 = %"PRId64"", i);
H
hzcheng 已提交
6138
      } else {
6139
        sprintf(tempS, " or t1 = %"PRId64" ", i);
H
hzcheng 已提交
6140
      }
6141
      strncat(condition, tempS, COND_BUF_LEN - 1);
H
hzcheng 已提交
6142

L
liu0x54 已提交
6143
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
6144 6145 6146 6147

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

6148
      double t = taosGetTimestampMs();
H
hzcheng 已提交
6149

S
Shuaiqiang Chang 已提交
6150 6151 6152 6153
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
6154
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6155
        taos_free_result(pSql);
H
hzcheng 已提交
6156
        taos_close(taos);
6157
        fclose(fp);
6158
        return NULL;
H
hzcheng 已提交
6159 6160
      }
      int count = 0;
6161
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6162 6163
        count++;
      }
6164
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6165

6166
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
6167 6168
              num_of_tables * num_of_DPT / (t * 1000.0), t);
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0);
H
hzcheng 已提交
6169

S
Shuaiqiang Chang 已提交
6170
      taos_free_result(pSql);
H
hzcheng 已提交
6171 6172 6173 6174
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
6175
#endif
H
hzcheng 已提交
6176 6177 6178
  return NULL;
}

6179 6180 6181 6182 6183 6184 6185
static void prompt()
{
  if (!g_args.answer_yes) {
    printf("         Press enter key to continue or Ctrl-C to stop\n\n");
    (void)getchar();
  }
}
H
Hui Li 已提交
6186

6187
static int insertTestProcess() {
6188

6189 6190 6191
  setupForAnsiEscape();
  int ret = printfInsertMeta();
  resetAfterAnsiEscape();
6192

6193 6194 6195
  if (ret == -1)
    exit(EXIT_FAILURE);

6196
  debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
6197 6198
  g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
  if (NULL == g_fpOfInsertResult) {
6199
    errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
6200 6201
    return -1;
  }
6202

6203 6204
  if (g_fpOfInsertResult)
    printfInsertMetaToFile(g_fpOfInsertResult);
6205

6206 6207 6208 6209
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void)getchar();
  }
6210

6211 6212 6213
  init_rand_data();

  // create database and super tables
6214
  if(createDatabasesAndStables() != 0) {
6215 6216
    if (g_fpOfInsertResult)
      fclose(g_fpOfInsertResult);
6217 6218
    return -1;
  }
6219 6220

  // pretreatement
6221
  prepareSampleData();
6222

6223 6224 6225 6226
  double start;
  double end;

  // create child tables
6227
  start = taosGetTimestampMs();
6228
  createChildTables();
6229
  end = taosGetTimestampMs();
6230

6231
  if (g_totalChildTables > 0) {
6232
    fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6233
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6234 6235
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
6236
            "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6237
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6238
    }
6239
  }
6240

6241
  taosMsleep(1000);
6242
  // create sub threads for inserting data
6243
  //start = taosGetTimestampMs();
6244
  for (int i = 0; i < g_Dbs.dbCount; i++) {
6245 6246
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
S
Shuduo Sang 已提交
6247
        for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6248 6249 6250 6251 6252

          SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];

          if (superTblInfo && (superTblInfo->insertRows > 0)) {
            startMultiThreadInsertData(
6253 6254 6255 6256
              g_Dbs.threadCount,
              g_Dbs.db[i].dbName,
              g_Dbs.db[i].dbCfg.precision,
              superTblInfo);
6257
          }
6258
        }
6259
      }
6260
    } else {
6261
        startMultiThreadInsertData(
6262 6263 6264
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
6265
          NULL);
H
Hui Li 已提交
6266
    }
6267
  }
6268
  //end = taosGetTimestampMs();
6269

6270
  //int64_t    totalInsertRows = 0;
6271
  //int64_t    totalAffectedRows = 0;
6272
  //for (int i = 0; i < g_Dbs.dbCount; i++) {
6273
  //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6274
  //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
6275 6276
  //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
  //}
6277
  //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
6278
  postFreeResource();
6279

6280 6281 6282
  return 0;
}

6283 6284
static void *specifiedTableQuery(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6285

6286
  if (pThreadInfo->taos == NULL) {
6287 6288
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
6289 6290 6291 6292
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
6293 6294
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6295
            pThreadInfo->threadID, taos_errstr(NULL));
6296 6297
      return NULL;
    } else {
6298
      pThreadInfo->taos = taos;
6299 6300 6301
    }
  }

6302 6303
  char sqlStr[MAX_DB_NAME_SIZE + 5];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6304 6305
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6306 6307 6308 6309
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
    return NULL;
  }
6310

6311 6312
  uint64_t st = 0;
  uint64_t et = 0;
6313

6314
  uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes;
6315

6316 6317 6318
  uint64_t totalQueried = 0;
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
6319

6320 6321 6322 6323 6324 6325
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
        sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }

6326
  while(queryTimes --) {
6327
    if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) <
6328
            (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) {
6329
      taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms
6330 6331
    }

6332 6333 6334
    st = taosGetTimestampMs();

    selectAndGetResult(pThreadInfo,
6335
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]);
6336

6337 6338 6339 6340
    et = taosGetTimestampMs();
    printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n",
              taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0);

6341 6342
    totalQueried ++;
    g_queryInfo.specifiedQueryInfo.totalQueried ++;
6343

6344 6345
    uint64_t  currentPrintTime = taosGetTimestampMs();
    uint64_t  endTs = taosGetTimestampMs();
6346
    if (currentPrintTime - lastPrintTime > 30*1000) {
6347
      debugPrint("%s() LN%d, endTs=%"PRIu64"ms, startTs=%"PRIu64"ms\n",
6348
          __func__, __LINE__, endTs, startTs);
6349
      printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.6f\n",
6350 6351
                    pThreadInfo->threadID,
                    totalQueried,
6352
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6353
      lastPrintTime = currentPrintTime;
6354
    }
H
Hui Li 已提交
6355
  }
6356 6357
  return NULL;
}
H
Hui Li 已提交
6358

6359
static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) {
6360 6361
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
6362 6363
  sprintf(subTblName, "%s.%s",
          g_queryInfo.dbName,
6364
          g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
6365 6366

  //printf("inSql: %s\n", inSql);
6367

6368 6369
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
6370
    return;
H
Hui Li 已提交
6371
  }
6372

6373
  tstrncpy(outSql, inSql, pos - inSql + 1);
6374
  //printf("1: %s\n", outSql);
6375
  strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1);
6376
  //printf("2: %s\n", outSql);
6377
  strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1);
6378
  //printf("3: %s\n", outSql);
H
Hui Li 已提交
6379 6380
}

6381
static void *superTableQuery(void *sarg) {
6382
  char sqlstr[MAX_QUERY_SQL_LENGTH];
6383
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6384

6385
  if (pThreadInfo->taos == NULL) {
6386 6387 6388 6389 6390 6391 6392 6393
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6394
            pThreadInfo->threadID, taos_errstr(NULL));
6395 6396
      return NULL;
    } else {
6397
      pThreadInfo->taos = taos;
6398 6399 6400
    }
  }

6401 6402
  uint64_t st = 0;
  uint64_t et = (int64_t)g_queryInfo.superQueryInfo.queryInterval;
6403

6404 6405 6406
  uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes;
  uint64_t totalQueried = 0;
  uint64_t  startTs = taosGetTimestampMs();
6407

6408
  uint64_t  lastPrintTime = taosGetTimestampMs();
6409
  while(queryTimes --) {
6410 6411 6412
    if (g_queryInfo.superQueryInfo.queryInterval
            && (et - st) < (int64_t)g_queryInfo.superQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.superQueryInfo.queryInterval - (et - st)); // ms
6413
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to);
6414
    }
H
Hui Li 已提交
6415

6416
    st = taosGetTimestampMs();
6417
    for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) {
6418
      for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
6419
        memset(sqlstr,0,sizeof(sqlstr));
6420
        replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i);
6421
        if (g_queryInfo.superQueryInfo.result[j][0] != 0) {
6422
          sprintf(pThreadInfo->fp, "%s-%d",
6423
                  g_queryInfo.superQueryInfo.result[j],
6424
                  pThreadInfo->threadID);
6425
        }
6426
        selectAndGetResult(pThreadInfo, sqlstr);
6427 6428 6429 6430 6431 6432 6433

        totalQueried++;
        g_queryInfo.superQueryInfo.totalQueried ++;

        int64_t  currentPrintTime = taosGetTimestampMs();
        int64_t  endTs = taosGetTimestampMs();
        if (currentPrintTime - lastPrintTime > 30*1000) {
6434
          printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.3f\n",
6435 6436
                    pThreadInfo->threadID,
                    totalQueried,
6437
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6438
          lastPrintTime = currentPrintTime;
6439
        }
6440
      }
H
Hui Li 已提交
6441
    }
6442
    et = taosGetTimestampMs();
6443
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%"PRIu64" - %"PRIu64"] once queries duration:%.4fs\n\n",
6444
            taosGetSelfPthreadId(),
6445 6446
            pThreadInfo->start_table_from,
            pThreadInfo->end_table_to,
6447
            (double)(et - st)/1000.0);
6448
  }
6449

6450 6451 6452
  return NULL;
}

6453
static int queryTestProcess() {
6454 6455 6456 6457

  setupForAnsiEscape();
  printfQueryMeta();
  resetAfterAnsiEscape();
6458 6459 6460 6461 6462 6463

  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
6464
          g_queryInfo.port);
6465
  if (taos == NULL) {
6466 6467
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6468 6469 6470
    exit(-1);
  }

6471
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6472 6473
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6474 6475 6476
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6477
  }
6478

6479 6480 6481 6482
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void)getchar();
  }
6483

6484 6485 6486
  if (g_args.debug_print || g_args.verbose_print) {
    printfQuerySystemInfo(taos);
  }
6487

6488 6489 6490 6491 6492 6493
  if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
    if (convertHostToServAddr(
        g_queryInfo.host, g_queryInfo.port, &g_queryInfo.serv_addr) != 0)
      exit(-1);
  }

6494 6495 6496
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
6497
  int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent;
6498
  uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount;
6499

6500
  uint64_t startTs = taosGetTimestampMs();
6501

6502 6503 6504 6505 6506 6507
  if ((nSqlCount > 0) && (nConcurrent > 0)) {

    pids  = malloc(nConcurrent * nSqlCount * sizeof(pthread_t));
    infos = malloc(nConcurrent * nSqlCount * sizeof(threadInfo));

    if ((NULL == pids) || (NULL == infos)) {
6508
      taos_close(taos);
6509
      ERROR_EXIT("memory allocation failed for create threads\n");
6510
    }
6511

6512 6513 6514 6515 6516 6517 6518 6519 6520 6521 6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 6532
    for (uint64_t i = 0; i < nSqlCount; i++) {
        for (int j = 0; j < nConcurrent; j++) {
            uint64_t seq = i * nConcurrent + j;
            threadInfo *t_info = infos + seq;
            t_info->threadID = seq;
            t_info->querySeq = i;

            if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {

                char sqlStr[MAX_TB_NAME_SIZE*2];
                sprintf(sqlStr, "use %s", g_queryInfo.dbName);
                verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
                if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) {
                    taos_close(taos);
                    free(infos);
                    free(pids);
                    errorPrint( "use database %s failed!\n\n",
                        g_queryInfo.dbName);
                    return -1;
                }
            }
6533

6534
            t_info->taos = NULL;// TODO: workaround to use separate taos connection;
6535

6536 6537
            pthread_create(pids + seq, NULL, specifiedTableQuery,
                t_info);
6538
        }
6539
    }
6540
  } else {
6541
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
6542
  }
6543

6544 6545
  taos_close(taos);

6546 6547 6548
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
6549 6550 6551 6552
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
    pidsOfSub  = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t));
    infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo));
6553 6554

    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
6555 6556
      free(infos);
      free(pids);
6557

6558
      ERROR_EXIT("memory allocation failed for create threads\n");
6559
    }
6560

S
Shuduo Sang 已提交
6561
    int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
6562
    int threads = g_queryInfo.superQueryInfo.threadCnt;
6563

S
Shuduo Sang 已提交
6564
    int64_t a = ntables / threads;
6565 6566 6567 6568
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
6569

S
Shuduo Sang 已提交
6570
    int64_t b = 0;
6571 6572 6573
    if (threads != 0) {
      b = ntables % threads;
    }
6574

6575
    uint64_t startFrom = 0;
6576
    for (int i = 0; i < threads; i++) {
6577 6578
      threadInfo *t_info = infosOfSub + i;
      t_info->threadID = i;
6579

6580 6581 6582 6583
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
6584
      t_info->taos = NULL; // TODO: workaround to use separate taos connection;
6585
      pthread_create(pidsOfSub + i, NULL, superTableQuery, t_info);
6586 6587
    }

6588
    g_queryInfo.superQueryInfo.threadCnt = threads;
6589
  } else {
6590
    g_queryInfo.superQueryInfo.threadCnt = 0;
6591
  }
6592

6593 6594 6595 6596 6597 6598
  if ((nSqlCount > 0) && (nConcurrent > 0)) {
    for (int i = 0; i < nConcurrent; i++) {
      for (int j = 0; j < nSqlCount; j++) {
        pthread_join(pids[i * nSqlCount + j], NULL);
      }
    }
6599
  }
S
Shuaiqiang Chang 已提交
6600

6601
  tmfree((char*)pids);
6602
  tmfree((char*)infos);
6603

6604
  for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
6605
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
6606
  }
H
Hui Li 已提交
6607

6608
  tmfree((char*)pidsOfSub);
6609
  tmfree((char*)infosOfSub);
6610

6611
//  taos_close(taos);// TODO: workaround to use separate taos connection;
6612
  uint64_t endTs = taosGetTimestampMs();
6613

6614
  uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried +
6615 6616
    g_queryInfo.superQueryInfo.totalQueried;

6617
  fprintf(stderr, "==== completed total queries: %"PRIu64", the QPS of all threads: %10.3f====\n",
6618
          totalQueried,
6619
          (double)(totalQueried/((endTs-startTs)/1000.0)));
6620 6621 6622
  return 0;
}

6623 6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637
static void stable_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
  if (res == NULL || taos_errno(res) != 0) {
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
    return;
  }

  if (param)
    appendResultToFile(res, ((threadInfo *)param)->fp);
  // tao_unscribe() will free result.
}

static void specified_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
6638
  if (res == NULL || taos_errno(res) != 0) {
6639 6640
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
6641 6642
    return;
  }
6643

6644
  if (param)
6645
    appendResultToFile(res, ((threadInfo *)param)->fp);
6646
  // tao_unscribe() will free result.
H
hzcheng 已提交
6647 6648
}

6649
static TAOS_SUB* subscribeImpl(
6650 6651 6652 6653
        QUERY_CLASS class,
        threadInfo *pThreadInfo,
        char *sql, char* topic, bool restart, uint64_t interval)
{
6654
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
6655

6656 6657 6658 6659
  if ((SPECIFIED_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) {
    tsub = taos_subscribe(
            pThreadInfo->taos,
6660
            restart,
6661
            topic, sql, specified_sub_callback, (void*)pThreadInfo,
6662
            g_queryInfo.specifiedQueryInfo.subscribeInterval);
6663 6664 6665 6666 6667 6668 6669
  } else if ((STABLE_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) {
    tsub = taos_subscribe(
            pThreadInfo->taos,
            restart,
            topic, sql, stable_sub_callback, (void*)pThreadInfo,
            g_queryInfo.superQueryInfo.subscribeInterval);
6670
  } else {
6671 6672
    tsub = taos_subscribe(
            pThreadInfo->taos,
6673
            restart,
6674
            topic, sql, NULL, NULL, interval);
6675
  }
6676

6677 6678 6679
  if (tsub == NULL) {
    printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
    return NULL;
6680
  }
6681

6682 6683
  return tsub;
}
H
hzcheng 已提交
6684

6685 6686
static void *superSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6687
  char subSqlstr[MAX_QUERY_SQL_LENGTH];
6688
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT] = {0};
H
hzcheng 已提交
6689

6690 6691
  if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
      errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n",
6692 6693 6694 6695 6696
              pThreadInfo->ntables,
              MAX_QUERY_SQL_COUNT);
      exit(-1);
  }

6697
  if (pThreadInfo->taos == NULL) {
6698 6699 6700 6701 6702 6703 6704 6705
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6706
            pThreadInfo->threadID, taos_errstr(NULL));
6707 6708
      return NULL;
    } else {
6709
      pThreadInfo->taos = taos;
6710 6711 6712
    }
  }

6713 6714
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6715 6716
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6717 6718
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
6719 6720
    return NULL;
  }
6721

S
Shengliang Guan 已提交
6722 6723 6724
  char topic[32] = {0};
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742 6743 6744 6745 6746 6747 6748 6749
      verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n",
              __func__, __LINE__,
              pThreadInfo->threadID,
              pThreadInfo->start_table_from,
              pThreadInfo->end_table_to, i);
      sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"",
              i, pThreadInfo->querySeq);
      memset(subSqlstr, 0, sizeof(subSqlstr));
      replaceChildTblName(
              g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq],
              subSqlstr, i);
      if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
        sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
      }

      debugPrint("%s() LN%d, [%d] subSqlstr: %s\n",
              __func__, __LINE__, pThreadInfo->threadID, subSqlstr);
      tsub[i] = subscribeImpl(
              STABLE_CLASS,
              pThreadInfo, subSqlstr, topic,
              g_queryInfo.superQueryInfo.subscribeRestart,
              g_queryInfo.superQueryInfo.subscribeInterval);
      if (NULL == tsub[i]) {
6750
        taos_close(pThreadInfo->taos);
6751 6752
        return NULL;
      }
S
Shengliang Guan 已提交
6753
  }
6754 6755

  // start loop to consume result
6756 6757 6758 6759
  int consumed[MAX_QUERY_SQL_COUNT];
  for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) {
    consumed[i] = 0;
  }
6760
  TAOS_RES* res = NULL;
6761

6762
  while(1) {
6763 6764
    for (uint64_t i = pThreadInfo->start_table_from;
            i <= pThreadInfo->end_table_to; i++) {
S
Shengliang Guan 已提交
6765 6766 6767
        if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
            continue;
        }
6768

6769
        res = taos_consume(tsub[i]);
S
Shengliang Guan 已提交
6770
        if (res) {
6771 6772
            if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
                sprintf(pThreadInfo->fp, "%s-%d",
6773 6774 6775 6776 6777 6778 6779 6780
                        g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                        pThreadInfo->threadID);
                appendResultToFile(res, pThreadInfo->fp);
            }
            if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
                sprintf(pThreadInfo->fp, "%s-%d",
                        g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                        pThreadInfo->threadID);
6781
                appendResultToFile(res, pThreadInfo->fp);
S
Shengliang Guan 已提交
6782
            }
6783
            consumed[i] ++;
6784 6785

            if ((g_queryInfo.superQueryInfo.subscribeKeepProgress)
6786 6787 6788 6789 6790 6791
                && (consumed[i] >=
                    g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
                printf("keepProgress:%d, resub super table query: %"PRIu64"\n",
                    g_queryInfo.superQueryInfo.subscribeKeepProgress,
                    pThreadInfo->querySeq);
                taos_unsubscribe(tsub,
6792
                    g_queryInfo.superQueryInfo.subscribeKeepProgress);
6793 6794 6795 6796 6797 6798 6799 6800
                consumed[i]= 0;
                tsub[i] = subscribeImpl(
                        STABLE_CLASS,
                        pThreadInfo, subSqlstr, topic,
                        g_queryInfo.superQueryInfo.subscribeRestart,
                        g_queryInfo.superQueryInfo.subscribeInterval
                    );
                if (NULL == tsub[i]) {
6801 6802 6803 6804
                    taos_close(pThreadInfo->taos);
                    return NULL;
                }
            }
6805
        }
H
hzcheng 已提交
6806 6807
    }
  }
6808
  taos_free_result(res);
6809

S
Shengliang Guan 已提交
6810 6811
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6812
    taos_unsubscribe(tsub[i], 0);
6813
  }
6814

6815
  taos_close(pThreadInfo->taos);
H
hzcheng 已提交
6816 6817 6818
  return NULL;
}

6819 6820
static void *specifiedSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6821
  TAOS_SUB*  tsub = NULL;
6822

6823
  if (pThreadInfo->taos == NULL) {
6824 6825 6826 6827 6828 6829 6830 6831
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6832
            pThreadInfo->threadID, taos_errstr(NULL));
6833 6834
      return NULL;
    } else {
6835
      pThreadInfo->taos = taos;
6836 6837 6838
    }
  }

6839 6840
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6841
  debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
6842 6843
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6844 6845
    return NULL;
  }
6846

S
Shengliang Guan 已提交
6847
  char topic[32] = {0};
6848 6849 6850 6851 6852 6853 6854 6855 6856 6857
  sprintf(topic, "taosdemo-subscribe-%"PRIu64"", pThreadInfo->querySeq);
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
      sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }
  tsub = subscribeImpl(
                      SPECIFIED_CLASS, pThreadInfo,
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
          topic,
6858
          g_queryInfo.specifiedQueryInfo.subscribeRestart,
6859 6860 6861 6862
          g_queryInfo.specifiedQueryInfo.subscribeInterval);
  if (NULL == tsub) {
      taos_close(pThreadInfo->taos);
      return NULL;
S
Shengliang Guan 已提交
6863
  }
6864

6865 6866
  // start loop to consume result
  TAOS_RES* res = NULL;
6867

6868
  int consumed;
6869

6870
  while(1) {
6871
      if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) {
6872 6873
        continue;
      }
6874

6875
      res = taos_consume(tsub);
6876
      if (res) {
6877 6878 6879 6880 6881 6882 6883
          if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
              sprintf(pThreadInfo->fp, "%s-%d",
                      g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
              appendResultToFile(res, pThreadInfo->fp);
          }

6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901
          consumed ++;
          if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress)
                && (consumed >=
                    g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
              printf("keepProgress:%d, resub specified query: %"PRIu64"\n",
                    g_queryInfo.specifiedQueryInfo.subscribeKeepProgress,
                    pThreadInfo->querySeq);
              consumed = 0;
              taos_unsubscribe(tsub,
                      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
              tsub = subscribeImpl(
                      SPECIFIED_CLASS,
                      pThreadInfo,
                      g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
                      topic,
                      g_queryInfo.specifiedQueryInfo.subscribeRestart,
                      g_queryInfo.specifiedQueryInfo.subscribeInterval);
              if (NULL == tsub) {
6902 6903
                taos_close(pThreadInfo->taos);
                return NULL;
6904 6905
              }
          }
6906 6907 6908
      }
  }
  taos_free_result(res);
6909
  taos_unsubscribe(tsub, 0);
6910
  taos_close(pThreadInfo->taos);
6911

H
hzcheng 已提交
6912 6913 6914
  return NULL;
}

6915
static int subscribeTestProcess() {
6916
  setupForAnsiEscape();
6917
  printfQueryMeta();
6918
  resetAfterAnsiEscape();
6919

6920 6921 6922 6923
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void) getchar();
  }
6924

6925 6926 6927 6928 6929
  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
6930
          g_queryInfo.port);
6931
  if (taos == NULL) {
6932 6933
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6934 6935 6936
    exit(-1);
  }

6937
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6938 6939
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6940 6941 6942
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6943 6944
  }

6945 6946
  taos_close(taos); // TODO: workaround to use separate taos connection;

6947 6948
  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
6949 6950 6951 6952 6953

  pthread_t  *pidsOfStable  = NULL;
  threadInfo *infosOfStable = NULL;

  //==== create threads for query for specified table
S
Shengliang Guan 已提交
6954
  if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) {
6955
    debugPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n",
6956
              __func__, __LINE__,
S
Shengliang Guan 已提交
6957 6958 6959 6960 6961 6962 6963 6964
              g_queryInfo.specifiedQueryInfo.sqlCount);
  } else {
    if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) {
        errorPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n",
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount);
        exit(-1);
    }
6965

6966 6967 6968 6969 6970 6971 6972 6973
    pids  = malloc(
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(pthread_t));
    infos = malloc(
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(threadInfo));
S
Shengliang Guan 已提交
6974 6975 6976 6977
    if ((NULL == pids) || (NULL == infos)) {
        errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
        exit(-1);
    }
6978

6979 6980 6981 6982 6983 6984 6985 6986 6987
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
            uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
            threadInfo *t_info = infos + seq;
            t_info->threadID = seq;
            t_info->querySeq = i;
            t_info->taos = NULL;  // TODO: workaround to use separate taos connection;
            pthread_create(pids + seq, NULL, specifiedSubscribe, t_info);
        }
S
Shengliang Guan 已提交
6988
    }
H
hzcheng 已提交
6989
  }
6990

6991 6992 6993 6994 6995 6996 6997
  //==== create threads for super table query
  if (g_queryInfo.superQueryInfo.sqlCount <= 0) {
    printf("%s() LN%d, super table query sqlCount %"PRIu64".\n",
              __func__, __LINE__,
              g_queryInfo.superQueryInfo.sqlCount);
  } else {
    if ((g_queryInfo.superQueryInfo.sqlCount > 0)
6998
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
6999 7000 7001
        pidsOfStable  = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
7002
            sizeof(pthread_t));
7003 7004 7005
        infosOfStable = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
7006
            sizeof(threadInfo));
7007 7008
        if ((NULL == pidsOfStable) || (NULL == infosOfStable)) {
            errorPrint("%s() LN%d, malloc failed for create threads\n",
7009
              __func__, __LINE__);
7010 7011 7012
            // taos_close(taos);
            exit(-1);
        }
7013

7014 7015
        int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
        int threads = g_queryInfo.superQueryInfo.threadCnt;
7016

7017 7018 7019 7020 7021
        int64_t a = ntables / threads;
        if (a < 1) {
            threads = ntables;
            a = 1;
        }
7022

7023 7024 7025 7026
        int64_t b = 0;
        if (threads != 0) {
            b = ntables % threads;
        }
7027

7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044
        for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            uint64_t startFrom = 0;
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                threadInfo *t_info = infosOfStable + seq;
                t_info->threadID = seq;
                t_info->querySeq = i;

                t_info->start_table_from = startFrom;
                t_info->ntables = j<b?a+1:a;
                t_info->end_table_to = j<b?startFrom+a:startFrom+a-1;
                startFrom = t_info->end_table_to + 1;
                t_info->taos = NULL; // TODO: workaround to use separate taos connection;
                pthread_create(pidsOfStable + seq,
                        NULL, superSubscribe, t_info);
            }
        }
7045

7046
        g_queryInfo.superQueryInfo.threadCnt = threads;
7047

7048 7049 7050 7051 7052 7053
        for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                pthread_join(pidsOfStable[seq], NULL);
            }
        }
7054
    }
H
hzcheng 已提交
7055
  }
7056

7057 7058 7059 7060 7061
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
    for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
        uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
        pthread_join(pids[seq], NULL);
    }
7062
  }
H
hzcheng 已提交
7063

7064
  tmfree((char*)pids);
7065
  tmfree((char*)infos);
H
hzcheng 已提交
7066

7067 7068
  tmfree((char*)pidsOfStable);
  tmfree((char*)infosOfStable);
7069
//   taos_close(taos);
7070
  return 0;
H
hzcheng 已提交
7071 7072
}

7073
static void initOfInsertMeta() {
7074
  memset(&g_Dbs, 0, sizeof(SDbs));
7075

7076
  // set default values
7077
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7078
  g_Dbs.port = 6030;
7079 7080
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
7081
  g_Dbs.threadCount = 2;
7082 7083

  g_Dbs.use_metric = g_args.use_metric;
L
Liu Tao 已提交
7084 7085
}

7086
static void initOfQueryMeta() {
7087
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
7088

7089
  // set default values
7090
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7091
  g_queryInfo.port = 6030;
7092 7093
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
L
Liu Tao 已提交
7094 7095
}

7096
static void setParaFromArg(){
7097
  if (g_args.host) {
7098
    tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE);
7099
  } else {
7100
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7101
  }
L
Liu Tao 已提交
7102

7103
  if (g_args.user) {
7104
    tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE);
7105
  }
7106 7107

  if (g_args.password) {
7108
    tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE);
7109 7110
  }

7111 7112
  if (g_args.port) {
    g_Dbs.port = g_args.port;
7113
  }
L
Liu Tao 已提交
7114

7115 7116 7117
  g_Dbs.threadCount = g_args.num_of_threads;
  g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;

7118 7119
  g_Dbs.dbCount = 1;
  g_Dbs.db[0].drop = 1;
7120

7121
  tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE);
7122
  g_Dbs.db[0].dbCfg.replica = g_args.replica;
7123
  tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
7124

7125
  tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
7126 7127 7128 7129 7130

  g_Dbs.use_metric = g_args.use_metric;
  g_Dbs.insert_only = g_args.insert_only;

  g_Dbs.do_aggreFunc = true;
L
Liu Tao 已提交
7131

7132 7133
  char dataString[STRING_LEN];
  char **data_type = g_args.datatype;
7134

7135
  memset(dataString, 0, STRING_LEN);
L
Liu Tao 已提交
7136

7137 7138
  if (strcasecmp(data_type[0], "BINARY") == 0
          || strcasecmp(data_type[0], "BOOL") == 0
7139
          || strcasecmp(data_type[0], "NCHAR") == 0 ) {
7140
    g_Dbs.do_aggreFunc = false;
L
Liu Tao 已提交
7141
  }
H
hzcheng 已提交
7142

7143 7144 7145 7146 7147
  if (g_args.use_metric) {
    g_Dbs.db[0].superTblCount = 1;
    tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE);
    g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
    g_Dbs.threadCount = g_args.num_of_threads;
7148
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
7149
    g_Dbs.asyncMode = g_args.async_mode;
7150

7151 7152 7153 7154
    g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
    g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
    g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
7155
    tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix,
7156 7157
            g_args.tb_prefix, MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
S
Shuduo Sang 已提交
7158
    g_Dbs.db[0].superTbls[0].insertMode = g_args.iface;
7159
    tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp,
7160
            "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
7161
    g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
7162

7163
    g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
7164
    g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len;
7165

7166 7167 7168 7169 7170
    g_Dbs.db[0].superTbls[0].columnCount = 0;
    for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
      if (data_type[i] == NULL) {
        break;
      }
7171

7172
      tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
7173
              data_type[i], MAX_TB_NAME_SIZE);
7174
      g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;
7175 7176
      g_Dbs.db[0].superTbls[0].columnCount++;
    }
7177

7178 7179 7180 7181 7182
    if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
      g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
    } else {
      for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) {
        tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE);
7183
        g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;
7184 7185 7186
        g_Dbs.db[0].superTbls[0].columnCount++;
      }
    }
7187

7188
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE);
7189
    g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;
7190

7191
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE);
7192 7193
    g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;
    g_Dbs.db[0].superTbls[0].tagCount = 2;
7194
  } else {
7195 7196
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
    g_Dbs.db[0].superTbls[0].tagCount = 0;
H
hzcheng 已提交
7197
  }
7198 7199 7200 7201 7202 7203
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
7204

7205 7206 7207
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
7208 7209 7210
    exit(-1);
  }

7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

7235
static void querySqlFile(TAOS* taos, char* sqlFile)
7236 7237 7238 7239 7240 7241
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
7242

7243 7244 7245 7246 7247 7248
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

7249
  double t = taosGetTimestampMs();
7250

7251
  while((read_len = tgetline(&line, &line_len, fp)) != -1) {
7252 7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
7264
    }
7265 7266

    memcpy(cmd + cmd_len, line, read_len);
7267
    verbosePrint("%s() LN%d cmd: %s\n", __func__, __LINE__, cmd);
7268 7269 7270
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) {
        errorPrint("%s() LN%d, queryDbExec %s failed!\n",
               __func__, __LINE__, cmd);
7271 7272 7273 7274 7275
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
7276 7277
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
7278 7279
  }

7280
  t = taosGetTimestampMs() - t;
7281
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
7282

7283 7284 7285 7286
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
7287 7288
}

7289
static void testMetaFile() {
7290
    if (INSERT_TEST == g_args.test_mode) {
7291 7292
      if (g_Dbs.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
7293

7294
      insertTestProcess();
7295

7296
    } else if (QUERY_TEST == g_args.test_mode) {
7297 7298
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7299

7300
      queryTestProcess();
7301

7302
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
7303 7304
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7305

7306
      subscribeTestProcess();
7307

7308 7309 7310
    }  else {
      ;
    }
7311
}
7312

7313
static void queryResult() {
7314
  // query data
7315

7316 7317 7318 7319 7320
  pthread_t read_id;
  threadInfo *rInfo = malloc(sizeof(threadInfo));
  assert(rInfo);
  rInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
  rInfo->start_table_from = 0;
7321

7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333 7334 7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356 7357
  //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
  if (g_args.use_metric) {
    rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
    rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
    rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
    tstrncpy(rInfo->tb_prefix,
          g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE);
  } else {
    rInfo->ntables = g_args.num_of_tables;
    rInfo->end_table_to = g_args.num_of_tables -1;
    tstrncpy(rInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE);
  }

  rInfo->taos = taos_connect(
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
          g_Dbs.port);
  if (rInfo->taos == NULL) {
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
    free(rInfo);
    exit(-1);
  }

  tstrncpy(rInfo->fp, g_Dbs.resultFile, MAX_FILE_NAME_LEN);

  if (!g_Dbs.use_metric) {
    pthread_create(&read_id, NULL, readTable, rInfo);
  } else {
    pthread_create(&read_id, NULL, readMetric, rInfo);
  }
  pthread_join(read_id, NULL);
  taos_close(rInfo->taos);
  free(rInfo);
7358 7359
}

7360 7361
static void testCmdLine() {

7362 7363 7364 7365 7366 7367 7368 7369 7370 7371
  if (strlen(configDir)) {
    wordexp_t full_path;
    if (wordexp(configDir, &full_path, 0) != 0) {
      errorPrint( "Invalid path %s\n", configDir);
      return;
    }
    taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
    wordfree(&full_path);
  }

7372 7373
  g_args.test_mode = INSERT_TEST;
  insertTestProcess();
7374

7375
  if (false == g_Dbs.insert_only)
7376
    queryResult();
7377 7378
}

7379 7380 7381
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

7382
  debugPrint("meta file: %s\n", g_args.metaFile);
7383 7384 7385

  if (g_args.metaFile) {
    initOfInsertMeta();
7386
    initOfQueryMeta();
7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
7400 7401 7402 7403
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
7404
          g_Dbs.port);
7405
      querySqlFile(qtaos, g_args.sqlFile);
7406 7407 7408 7409 7410
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
7411 7412 7413

    if (g_dupstr)
        free(g_dupstr);
H
hzcheng 已提交
7414
  }
7415 7416

  return 0;
H
hzcheng 已提交
7417
}
7418