taosdemo.c 247.3 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

21
#include <stdint.h>
S
slguan 已提交
22
#define _GNU_SOURCE
23 24
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
41
#else
S
TD-1057  
Shengliang Guan 已提交
42 43
  #include <regex.h>
  #include <stdio.h>
44
#endif
H
hzcheng 已提交
45

46
#include <assert.h>
47
#include <stdlib.h>
48 49
#include "cJSON.h"

50
#include "os.h"
H
hzcheng 已提交
51
#include "taos.h"
52
#include "taoserror.h"
H
Hui Li 已提交
53
#include "tutil.h"
S
slguan 已提交
54

55 56 57
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
58 59
extern char configDir[];

60 61 62 63
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

64 65
#define STR_INSERT_INTO     "INSERT INTO "

66 67 68 69 70 71
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};
72

73 74
#define MAX_RECORDS_PER_REQ     32766

75 76
#define HEAD_BUFF_LEN    1024*24  // 16*1024 + (192+32)*2 + insert into ..

77 78
#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
79
#define COND_BUF_LEN        BUFFER_SIZE - 30
80 81
#define MAX_USERNAME_SIZE  64
#define MAX_PASSWORD_SIZE  64
82
#define MAX_DB_NAME_SIZE   64
83
#define MAX_HOSTNAME_SIZE  64
84
#define MAX_TB_NAME_SIZE   64
85
#define MAX_DATA_SIZE      (16*1024)+20     // max record len: 16*1024, timestamp string and ,('') need extra space
86 87 88 89 90 91 92 93 94 95
#define MAX_NUM_DATATYPE   10
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
#define MAX_FILE_NAME_LEN  256

#define   MAX_SAMPLES_ONCE_FROM_FILE   10000
#define   MAX_NUM_DATATYPE 10

#define   MAX_DB_COUNT           8
96
#define   MAX_SUPER_TABLE_COUNT  200
97 98 99
#define   MAX_COLUMN_COUNT       1024
#define   MAX_TAG_COUNT          128

100
#define   MAX_QUERY_SQL_COUNT    100
101
#define   MAX_QUERY_SQL_LENGTH   1024
102 103

#define   MAX_DATABASE_COUNT     256
104
#define INPUT_BUF_LEN   256
105

106
#define DEFAULT_TIMESTAMP_STEP  1
107

108

109 110 111 112 113
typedef enum CREATE_SUB_TALBE_MOD_EN {
  PRE_CREATE_SUBTBL,
  AUTO_CREATE_SUBTBL,
  NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
114

115 116
typedef enum TALBE_EXISTS_EN {
  TBL_NO_EXISTS,
117
  TBL_ALREADY_EXISTS,
118 119 120
  TBL_EXISTS_BUTT
} TALBE_EXISTS_EN;

121
enum enumSYNC_MODE {
122 123
  SYNC_MODE,
  ASYNC_MODE,
124 125
  MODE_BUT
};
126

S
Shuduo Sang 已提交
127
enum enum_TAOS_INTERFACE {
128 129 130
    TAOSC_IFACE,
    REST_IFACE,
    STMT_IFACE,
131 132 133
    INTERFACE_BUT
};

134 135 136 137 138 139
typedef enum enumQUERY_CLASS {
    SPECIFIED_CLASS,
    STABLE_CLASS,
    CLASS_BUT
} QUERY_CLASS;

S
Shuduo Sang 已提交
140
typedef enum enum_PROGRESSIVE_OR_INTERLACE {
141 142 143
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
144
} PROG_OR_INTERLACE_MODE;
145

146
typedef enum enumQUERY_TYPE {
147
  NO_INSERT_TYPE,
148
  INSERT_TYPE,
149
  QUERY_TYPE_BUT
150
} QUERY_TYPE;
151 152 153 154 155 156 157

enum _show_db_index {
  TSDB_SHOW_DB_NAME_INDEX,
  TSDB_SHOW_DB_CREATED_TIME_INDEX,
  TSDB_SHOW_DB_NTABLES_INDEX,
  TSDB_SHOW_DB_VGROUPS_INDEX,
  TSDB_SHOW_DB_REPLICA_INDEX,
158
  TSDB_SHOW_DB_QUORUM_INDEX,
159 160 161 162 163 164 165 166 167 168
  TSDB_SHOW_DB_DAYS_INDEX,
  TSDB_SHOW_DB_KEEP_INDEX,
  TSDB_SHOW_DB_CACHE_INDEX,
  TSDB_SHOW_DB_BLOCKS_INDEX,
  TSDB_SHOW_DB_MINROWS_INDEX,
  TSDB_SHOW_DB_MAXROWS_INDEX,
  TSDB_SHOW_DB_WALLEVEL_INDEX,
  TSDB_SHOW_DB_FSYNC_INDEX,
  TSDB_SHOW_DB_COMP_INDEX,
  TSDB_SHOW_DB_CACHELAST_INDEX,
169
  TSDB_SHOW_DB_PRECISION_INDEX,
170 171 172 173 174 175 176 177 178 179
  TSDB_SHOW_DB_UPDATE_INDEX,
  TSDB_SHOW_DB_STATUS_INDEX,
  TSDB_MAX_SHOW_DB
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
  TSDB_SHOW_STABLES_NAME_INDEX,
  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
  TSDB_SHOW_STABLES_COLUMNS_INDEX,
180 181
  TSDB_SHOW_STABLES_METRIC_INDEX,
  TSDB_SHOW_STABLES_UID_INDEX,
182
  TSDB_SHOW_STABLES_TID_INDEX,
183
  TSDB_SHOW_STABLES_VGID_INDEX,
184
  TSDB_MAX_SHOW_STABLES
185 186
};

187 188 189 190 191 192 193 194 195 196 197 198 199 200
enum _describe_table_index {
  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
  TSDB_MAX_DESCRIBE_METRIC
};

typedef struct {
  char field[TSDB_COL_NAME_LEN + 1];
  char type[16];
  int length;
  char note[128];
} SColDes;
H
hzcheng 已提交
201 202

/* Used by main to communicate with parse_opt. */
203 204
static char *g_dupstr = NULL;

205 206
typedef struct SArguments_S {
  char *   metaFile;
207
  uint32_t test_mode;
208 209
  char *   host;
  uint16_t port;
S
Shuduo Sang 已提交
210
  uint16_t iface;
211 212 213 214 215 216 217
  char *   user;
  char *   password;
  char *   database;
  int      replica;
  char *   tb_prefix;
  char *   sqlFile;
  bool     use_metric;
218
  bool     drop_database;
219
  bool     insert_only;
220
  bool     answer_yes;
221
  bool     debug_print;
222
  bool     verbose_print;
223
  bool     performance_print;
224
  char *   output_file;
225
  bool     async_mode;
226
  char *   datatype[MAX_NUM_DATATYPE + 1];
227 228 229 230
  uint32_t len_of_binary;
  uint32_t num_of_CPR;
  uint32_t num_of_threads;
  uint64_t insert_interval;
231
  int64_t  query_times;
232 233 234
  uint64_t interlace_rows;
  uint64_t num_of_RPR;                  // num_of_records_per_req
  uint64_t max_sql_len;
235 236
  int64_t  num_of_tables;
  int64_t  num_of_DPT;
237
  int      abort;
238 239
  int      disorderRatio;               // 0: no disorder, >0: x%
  int      disorderRange;               // ms or us by database precision
240
  uint32_t method_of_delete;
241
  char **  arg_list;
242 243
  uint64_t totalInsertRows;
  uint64_t totalAffectedRows;
244 245 246
} SArguments;

typedef struct SColumn_S {
247 248 249
  char      field[TSDB_COL_NAME_LEN + 1];
  char      dataType[MAX_TB_NAME_SIZE];
  uint32_t  dataLen;
250
  char      note[128];
251 252 253
} StrColumn;

typedef struct SSuperTable_S {
254
  char         sTblName[MAX_TB_NAME_SIZE+1];
255
  int64_t      childTblCount;
256
  bool         childTblExists;          // 0: no, 1: yes
257 258
  uint64_t     batchCreateTableNum;     // 0: no batch,  > 0: batch table number in one sql
  uint8_t      autoCreateTable;         // 0: create sub table, 1: auto create sub table
259
  char         childTblPrefix[MAX_TB_NAME_SIZE];
260
  char         dataSource[MAX_TB_NAME_SIZE+1];  // rand_gen or sample
261
  uint16_t     insertMode;              // 0: taosc, 1: rest, 2: stmt
262
  int64_t      childTblLimit;
263
  uint64_t     childTblOffset;
264

265
//  int          multiThreadWriteOneTbl;  // 0: no, 1: yes
266
  uint64_t     interlaceRows;           //
267 268
  int          disorderRatio;           // 0: no disorder, >0: x%
  int          disorderRange;           // ms or us by database precision
269
  uint64_t     maxSqlLen;               //
270

271
  uint64_t     insertInterval;          // insert interval, will override global insert interval
272
  int64_t      insertRows;
273
  int64_t      timeStampStep;
274
  char         startTimestamp[MAX_TB_NAME_SIZE];
275
  char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
276 277
  char         sampleFile[MAX_FILE_NAME_LEN+1];
  char         tagsFile[MAX_FILE_NAME_LEN+1];
278

279
  uint32_t     columnCount;
280
  StrColumn    columns[MAX_COLUMN_COUNT];
281
  uint32_t     tagCount;
282 283 284
  StrColumn    tags[MAX_TAG_COUNT];

  char*        childTblName;
285
  char*        colsOfCreateChildTable;
286 287
  uint64_t     lenOfOneRow;
  uint64_t     lenOfTagOfOneRow;
288 289 290 291 292

  char*        sampleDataBuf;
  //int          sampleRowCount;
  //int          sampleUsePos;

293
  uint32_t     tagSource;    // 0: rand, 1: tag sample
294
  char*        tagDataBuf;
295 296
  uint32_t     tagSampleCount;
  uint32_t     tagUsePos;
297

298
  // statistics
299 300
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
301 302 303 304 305
} SSuperTable;

typedef struct {
  char     name[TSDB_DB_NAME_LEN + 1];
  char     create_time[32];
S
Shuduo Sang 已提交
306
  int64_t  ntables;
307
  int32_t  vgroups;
308 309
  int16_t  replica;
  int16_t  quorum;
310
  int16_t  days;
311 312 313 314 315 316 317 318 319 320 321 322 323 324
  char     keeplist[32];
  int32_t  cache; //MB
  int32_t  blocks;
  int32_t  minrows;
  int32_t  maxrows;
  int8_t   wallevel;
  int32_t  fsync;
  int8_t   comp;
  int8_t   cachelast;
  char     precision[8];   // time resolution
  int8_t   update;
  char     status[16];
} SDbInfo;

325
typedef struct SDbCfg_S {
326
//  int       maxtablesPerVnode;
327 328
  uint32_t  minRows;        // 0 means default
  uint32_t  maxRows;        // 0 means default
329 330
  int       comp;
  int       walLevel;
331
  int       cacheLast;
332
  int       fsync;
333 334 335 336 337 338 339
  int       replica;
  int       update;
  int       keep;
  int       days;
  int       cache;
  int       blocks;
  int       quorum;
340
  char      precision[MAX_TB_NAME_SIZE];
341 342 343 344
} SDbCfg;

typedef struct SDataBase_S {
  char         dbName[MAX_DB_NAME_SIZE];
345
  bool         drop;  // 0: use exists, 1: if exists, drop then new create
346
  SDbCfg       dbCfg;
347
  uint64_t     superTblCount;
348 349 350 351
  SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
} SDataBase;

typedef struct SDbs_S {
352
  char         cfgDir[MAX_FILE_NAME_LEN+1];
353
  char         host[MAX_HOSTNAME_SIZE];
354 355
  struct sockaddr_in serv_addr;

356
  uint16_t     port;
357 358
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
359
  char         resultFile[MAX_FILE_NAME_LEN+1];
360 361 362
  bool         use_metric;
  bool         insert_only;
  bool         do_aggreFunc;
363
  bool         asyncMode;
364

365 366 367
  uint32_t     threadCount;
  uint32_t     threadCountByCreateTbl;
  uint32_t     dbCount;
368 369 370
  SDataBase    db[MAX_DB_COUNT];

  // statistics
371 372
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
373

374 375
} SDbs;

376
typedef struct SpecifiedQueryInfo_S {
377 378 379
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint64_t     concurrent;
  uint64_t     sqlCount;
380
  uint32_t     asyncMode; // 0: sync, 1: async
381 382
  uint64_t     subscribeInterval; // ms
  uint64_t     queryTimes;
383
  bool         subscribeRestart;
384
  int          subscribeKeepProgress;
385
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
386
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
387
  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
388
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
389
  uint64_t     totalQueried;
390
} SpecifiedQueryInfo;
391

392
typedef struct SuperQueryInfo_S {
393
  char         sTblName[MAX_TB_NAME_SIZE+1];
394 395
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint32_t     threadCnt;
396
  uint32_t     asyncMode; // 0: sync, 1: async
397
  uint64_t     subscribeInterval; // ms
398
  bool         subscribeRestart;
399
  int          subscribeKeepProgress;
400
  uint64_t     queryTimes;
401
  int64_t      childTblCount;
402
  char         childTblPrefix[MAX_TB_NAME_SIZE];
403
  uint64_t     sqlCount;
404
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
405
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
406
  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
407
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
408

409
  char*        childTblName;
410
  uint64_t     totalQueried;
411
} SuperQueryInfo;
412 413

typedef struct SQueryMetaInfo_S {
414
  char         cfgDir[MAX_FILE_NAME_LEN+1];
415
  char         host[MAX_HOSTNAME_SIZE];
416
  uint16_t     port;
417
  struct       sockaddr_in serv_addr;
418 419
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
420
  char         dbName[MAX_DB_NAME_SIZE+1];
421
  char         queryMode[MAX_TB_NAME_SIZE];  // taosc, rest
422

423
  SpecifiedQueryInfo  specifiedQueryInfo;
424
  SuperQueryInfo      superQueryInfo;
425
  uint64_t     totalQueried;
426 427 428
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
429
  TAOS *    taos;
430
  TAOS_STMT *stmt;
431 432 433 434 435 436 437
  int       threadID;
  char      db_name[MAX_DB_NAME_SIZE+1];
  uint32_t  time_precision;
  char      fp[4096];
  char      tb_prefix[MAX_TB_NAME_SIZE];
  uint64_t  start_table_from;
  uint64_t  end_table_to;
S
Shuduo Sang 已提交
438
  int64_t   ntables;
439 440 441 442
  uint64_t  data_of_rate;
  int64_t   start_time;
  char*     cols;
  bool      use_metric;
443
  SSuperTable* superTblInfo;
S
Shuduo Sang 已提交
444
  char      *buffer;    // sql cmd buffer
445 446

  // for async insert
447 448
  tsem_t    lock_sem;
  int64_t   counter;
449 450
  uint64_t  st;
  uint64_t  et;
451
  uint64_t  lastTs;
452

453
  // sample data
454
  int64_t   samplePos;
455
  // statistics
456 457
  uint64_t  totalInsertRows;
  uint64_t  totalAffectedRows;
458 459

  // insert delay statistics
460 461 462 463 464
  uint64_t  cntDelay;
  uint64_t  totalDelay;
  uint64_t  avgDelay;
  uint64_t  maxDelay;
  uint64_t  minDelay;
465

466
  // seq of query or subscribe
467
  uint64_t  querySeq;   // sequence number of sql command
468

469 470
} threadInfo;

471
#ifdef WINDOWS
472 473
#define _CRT_RAND_S

474
#include <windows.h>
475 476 477 478 479
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
480 481
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
482 483
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
484 485 486 487

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

488
static void setupForAnsiEscape(void) {
489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
507
  }
508 509
}

510
static void resetAfterAnsiEscape(void) {
511
  // Reset colors
512
  printf("\x1b[0m");
513 514 515 516 517 518

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
519

520
static int taosRandom()
521 522 523 524 525 526
{
    int number;
    rand_s(&number);

    return number;
}
527
#else   // Not windows
528
static void setupForAnsiEscape(void) {}
529

530
static void resetAfterAnsiEscape(void) {
531 532 533
  // Reset colors
  printf("\x1b[0m");
}
534

535 536
#include <time.h>

537
static int taosRandom()
538
{
539
  return rand();
540 541
}

542
#endif // ifdef Windows
543

544
static void prompt();
545
static int createDatabasesAndStables();
546
static void createChildTables();
547
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet);
548 549
static int postProceSql(char *host, struct sockaddr_in *pServAddr,
        uint16_t port, char* sqlstr, char *resultFile);
550 551 552 553 554 555 556

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
557
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)",
558 559
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

560
SArguments g_args = {
561 562 563 564
                     NULL,            // metaFile
                     0,               // test_mode
                     "127.0.0.1",     // host
                     6030,            // port
S
Shuduo Sang 已提交
565
                     TAOSC_IFACE,     // iface
566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597
                     "root",          // user
                     #ifdef _TD_POWER_
                     "powerdb",      // password
                     #else
                     "taosdata",      // password
                     #endif
                     "test",          // database
                     1,               // replica
                     "t",             // tb_prefix
                     NULL,            // sqlFile
                     true,            // use_metric
                     true,            // drop_database
                     true,            // insert_only
                     false,           // debug_print
                     false,           // verbose_print
                     false,           // performance statistic print
                     false,           // answer_yes;
                     "./output.txt",  // output_file
                     0,               // mode : sync or async
                     {
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     },
                     16,              // len_of_binary
                     4,               // num_of_CPR
                     10,              // num_of_connections/thread
                     0,               // insert_interval
                     1,               // query_times
                     0,               // interlace_rows;
                     30000,           // num_of_RPR
sangshuduo's avatar
sangshuduo 已提交
598
                     (1024*1024),         // max_sql_len
599 600 601 602 603 604 605
                     10000,           // num_of_tables
                     10000,           // num_of_DPT
                     0,               // abort
                     0,               // disorderRatio
                     1000,            // disorderRange
                     1,               // method_of_delete
                     NULL             // arg_list
606 607 608 609 610
};



static SDbs            g_Dbs;
611
static int64_t         g_totalChildTables = 0;
612 613 614 615
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

#define debugPrint(fmt, ...) \
616
    do { if (g_args.debug_print || g_args.verbose_print) \
617
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
618

619
#define verbosePrint(fmt, ...) \
620 621
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
622

623 624 625 626
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)

627 628 629 630
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)


631
///////////////////////////////////////////////////
H
hzcheng 已提交
632

633 634
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

635 636 637 638 639 640 641 642
#ifndef TAOSDEMO_COMMIT_SHA1
#define TAOSDEMO_COMMIT_SHA1 "unknown"
#endif

#ifndef TD_VERNUMBER
#define TD_VERNUMBER    "unknown"
#endif

643
#ifndef TAOSDEMO_STATUS
644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660
#define TAOSDEMO_STATUS "unknown"
#endif

static void printVersion() {
    char tdengine_ver[] = TD_VERNUMBER;
    char taosdemo_ver[] = TAOSDEMO_COMMIT_SHA1;
    char taosdemo_status[] = TAOSDEMO_STATUS;

    if (strlen(taosdemo_status) == 0) {
        printf("taosdemo verison %s-%s\n",
                tdengine_ver, taosdemo_ver);
    } else {
        printf("taosdemo verison %s-%s, status:%s\n",
                tdengine_ver, taosdemo_ver, taosdemo_status);
    }
}

661
static void printHelp() {
662
  char indent[10] = "        ";
663
  printf("%s%s%s%s\n", indent, "-f", indent,
664
          "The meta file to the execution procedure. Default is './meta.json'.");
665
  printf("%s%s%s%s\n", indent, "-u", indent,
666 667
          "The TDengine user name to use when connecting to the server. Default is 'root'.");
#ifdef _TD_POWER_
668
  printf("%s%s%s%s\n", indent, "-P", indent,
669
          "The password to use when connecting to the server. Default is 'powerdb'.");
670
  printf("%s%s%s%s\n", indent, "-c", indent,
671 672
          "Configuration directory. Default is '/etc/power/'.");
#else
673
  printf("%s%s%s%s\n", indent, "-P", indent,
674
          "The password to use when connecting to the server. Default is 'taosdata'.");
675
  printf("%s%s%s%s\n", indent, "-c", indent,
676
          "Configuration directory. Default is '/etc/taos/'.");
677
#endif
678
  printf("%s%s%s%s\n", indent, "-h", indent,
679
          "The host to connect to TDengine. Default is localhost.");
680
  printf("%s%s%s%s\n", indent, "-p", indent,
681
          "The TCP/IP port number to use for the connection. Default is 0.");
682 683
  printf("%s%s%s%s\n", indent, "-I", indent,
          "The interface (taosc, rest, and stmt) taosdemo uses. Default is 'taosc'.");
684
  printf("%s%s%s%s\n", indent, "-d", indent,
685
          "Destination database. Default is 'test'.");
686
  printf("%s%s%s%s\n", indent, "-a", indent,
687
          "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
688
  printf("%s%s%s%s\n", indent, "-m", indent,
689 690
          "Table prefix name. Default is 't'.");
  printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
691
  printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
692
  printf("%s%s%s%s\n", indent, "-o", indent,
693
          "Direct output to the named file. Default is './output.txt'.");
694
  printf("%s%s%s%s\n", indent, "-q", indent,
695
          "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC.");
696
  printf("%s%s%s%s\n", indent, "-b", indent,
sangshuduo's avatar
sangshuduo 已提交
697
          "The data_type of columns, default: INT,INT,INT,INT.");
698
  printf("%s%s%s%s\n", indent, "-w", indent,
699
          "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
700 701 702
  printf("%s%s%s%s%d\n", indent, "-l", indent,
          "The number of columns per record. Default is 4. Max values is ",
       MAX_NUM_DATATYPE);
703
  printf("%s%s%s%s\n", indent, "-T", indent,
704
          "The number of threads. Default is 10.");
705
  printf("%s%s%s%s\n", indent, "-i", indent,
706
          "The sleep time (ms) between insertion. Default is 0.");
707
  printf("%s%s%s%s\n", indent, "-r", indent,
708
          "The number of records per request. Default is 30000.");
709
  printf("%s%s%s%s\n", indent, "-t", indent,
710
          "The number of tables. Default is 10000.");
711
  printf("%s%s%s%s\n", indent, "-n", indent,
712 713 714
          "The number of records per table. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
  printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
715
  printf("%s%s%s%s\n", indent, "-O", indent,
716
          "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order.");
717
  printf("%s%s%s%s\n", indent, "-R", indent,
718
          "Out of order data's range, ms, default is 1000.");
719
  printf("%s%s%s%s\n", indent, "-g", indent,
720
          "Print debug info.");
721
  printf("%s%s%s\n", indent, "-V, --version\t",
722
          "Print version info.");
723 724
  printf("%s%s%s%s\n", indent, "--help\t", indent,
          "Print command line arguments list info.");
725
/*    printf("%s%s%s%s\n", indent, "-D", indent,
726 727 728 729
          "if elete database if exists. 0: no, 1: yes, default is 1");
          */
}

730 731 732 733 734 735 736 737 738 739 740 741 742 743 744
static bool isStringNumber(char *input)
{
  int len = strlen(input);
  if (0 == len) {
    return false;
  }

  for (int i = 0; i < len; i++) {
    if (!isdigit(input[i]))
      return false;
  }

  return true;
}

745
static void parse_args(int argc, char *argv[], SArguments *arguments) {
746 747 748 749 750

  for (int i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-f") == 0) {
      arguments->metaFile = argv[++i];
    } else if (strcmp(argv[i], "-c") == 0) {
H
Haojun Liao 已提交
751 752 753 754 755
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-c need a valid path following!\n");
        exit(EXIT_FAILURE);
      }
756
      tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN);
757
    } else if (strcmp(argv[i], "-h") == 0) {
H
Haojun Liao 已提交
758 759 760 761 762
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-h need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
763 764
      arguments->host = argv[++i];
    } else if (strcmp(argv[i], "-p") == 0) {
765
      if ((argc == i+1) ||
766
        (!isStringNumber(argv[i+1]))) {
767 768 769 770
        printHelp();
        errorPrint("%s", "\n\t-p need a number following!\n");
        exit(EXIT_FAILURE);
      }
771
      arguments->port = atoi(argv[++i]);
772 773 774 775 776 777 778 779
    } else if (strcmp(argv[i], "-I") == 0) {
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-I need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
      ++i;
      if (0 == strcasecmp(argv[i], "taosc")) {
S
Shuduo Sang 已提交
780
          arguments->iface = TAOSC_IFACE;
781
      } else if (0 == strcasecmp(argv[i], "rest")) {
S
Shuduo Sang 已提交
782
          arguments->iface = REST_IFACE;
783
      } else if (0 == strcasecmp(argv[i], "stmt")) {
S
Shuduo Sang 已提交
784
          arguments->iface = STMT_IFACE;
785 786 787 788
      } else {
        errorPrint("%s", "\n\t-I need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
789
    } else if (strcmp(argv[i], "-u") == 0) {
H
Haojun Liao 已提交
790 791 792 793 794
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-u need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
795 796
      arguments->user = argv[++i];
    } else if (strcmp(argv[i], "-P") == 0) {
H
Haojun Liao 已提交
797 798 799 800 801
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-P need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
802 803
      arguments->password = argv[++i];
    } else if (strcmp(argv[i], "-o") == 0) {
H
Haojun Liao 已提交
804 805 806 807 808
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-o need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
809 810
      arguments->output_file = argv[++i];
    } else if (strcmp(argv[i], "-s") == 0) {
H
Haojun Liao 已提交
811 812 813 814 815
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-s need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
816 817
      arguments->sqlFile = argv[++i];
    } else if (strcmp(argv[i], "-q") == 0) {
818 819
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
820
        printHelp();
821
        errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n");
822 823
        exit(EXIT_FAILURE);
      }
824
      arguments->async_mode = atoi(argv[++i]);
825
    } else if (strcmp(argv[i], "-T") == 0) {
826 827
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
828 829 830 831
        printHelp();
        errorPrint("%s", "\n\t-T need a number following!\n");
        exit(EXIT_FAILURE);
      }
832 833
      arguments->num_of_threads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
834 835
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
836 837 838 839
        printHelp();
        errorPrint("%s", "\n\t-i need a number following!\n");
        exit(EXIT_FAILURE);
      }
840
      arguments->insert_interval = atoi(argv[++i]);
841
    } else if (strcmp(argv[i], "-qt") == 0) {
842
      if ((argc == i+1)
843
              || (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
844
        printHelp();
845
        errorPrint("%s", "\n\t-qt need a number following!\n");
H
Haojun Liao 已提交
846 847
        exit(EXIT_FAILURE);
      }
848
      arguments->query_times = atoi(argv[++i]);
849
    } else if (strcmp(argv[i], "-B") == 0) {
850 851
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
852 853 854 855
        printHelp();
        errorPrint("%s", "\n\t-B need a number following!\n");
        exit(EXIT_FAILURE);
      }
856
      arguments->interlace_rows = atoi(argv[++i]);
857
    } else if (strcmp(argv[i], "-r") == 0) {
858 859
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
860 861 862 863
        printHelp();
        errorPrint("%s", "\n\t-r need a number following!\n");
        exit(EXIT_FAILURE);
      }
864 865
      arguments->num_of_RPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-t") == 0) {
866 867 868 869 870 871
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-t need a number following!\n");
        exit(EXIT_FAILURE);
      }
872 873
      arguments->num_of_tables = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
874 875 876 877 878 879
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-n need a number following!\n");
        exit(EXIT_FAILURE);
      }
880 881
      arguments->num_of_DPT = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-d") == 0) {
H
Haojun Liao 已提交
882 883 884 885 886
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-d need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
887 888
      arguments->database = argv[++i];
    } else if (strcmp(argv[i], "-l") == 0) {
889 890 891 892 893 894
      if (argc == i+1) {
        if (!isStringNumber(argv[i+1])) {
            printHelp();
            errorPrint("%s", "\n\t-l need a number following!\n");
            exit(EXIT_FAILURE);
        }
895
      }
896
      arguments->num_of_CPR = atoi(argv[++i]);
897 898 899 900 901 902 903 904 905 906 907

      if (arguments->num_of_CPR > MAX_NUM_DATATYPE) {
          printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE);
          prompt();
          arguments->num_of_CPR = MAX_NUM_DATATYPE;
      }

      for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) {
          arguments->datatype[col] = NULL;
      }

908
    } else if (strcmp(argv[i], "-b") == 0) {
909 910 911 912 913
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-b need valid string following!\n");
        exit(EXIT_FAILURE);
      }
914 915
      ++i;
      if (strstr(argv[i], ",") == NULL) {
916
        // only one col
917
        if (strcasecmp(argv[i], "INT")
918 919 920 921 922 923 924 925
                && strcasecmp(argv[i], "FLOAT")
                && strcasecmp(argv[i], "TINYINT")
                && strcasecmp(argv[i], "BOOL")
                && strcasecmp(argv[i], "SMALLINT")
                && strcasecmp(argv[i], "BIGINT")
                && strcasecmp(argv[i], "DOUBLE")
                && strcasecmp(argv[i], "BINARY")
                && strcasecmp(argv[i], "NCHAR")) {
926
          printHelp();
927
          errorPrint("%s", "-b: Invalid data_type!\n");
928
          exit(EXIT_FAILURE);
929
        }
930
        arguments->datatype[0] = argv[i];
931
      } else {
932
        // more than one col
933
        int index = 0;
934 935
        g_dupstr = strdup(argv[i]);
        char *running = g_dupstr;
936
        char *token = strsep(&running, ",");
937
        while(token != NULL) {
938
          if (strcasecmp(token, "INT")
939 940 941 942 943 944 945 946
                  && strcasecmp(token, "FLOAT")
                  && strcasecmp(token, "TINYINT")
                  && strcasecmp(token, "BOOL")
                  && strcasecmp(token, "SMALLINT")
                  && strcasecmp(token, "BIGINT")
                  && strcasecmp(token, "DOUBLE")
                  && strcasecmp(token, "BINARY")
                  && strcasecmp(token, "NCHAR")) {
947
            printHelp();
948
            free(g_dupstr);
949
            errorPrint("%s", "-b: Invalid data_type!\n");
950
            exit(EXIT_FAILURE);
H
hzcheng 已提交
951
          }
952
          arguments->datatype[index++] = token;
953 954
          token = strsep(&running, ",");
          if (index >= MAX_NUM_DATATYPE) break;
S
TD-1057  
Shengliang Guan 已提交
955
        }
956
        arguments->datatype[index] = NULL;
957
      }
958
    } else if (strcmp(argv[i], "-w") == 0) {
959
      if ((argc == i+1) ||
960
        (!isStringNumber(argv[i+1]))) {
961 962 963 964
        printHelp();
        errorPrint("%s", "\n\t-w need a number following!\n");
        exit(EXIT_FAILURE);
      }
965 966
      arguments->len_of_binary = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-m") == 0) {
H
Haojun Liao 已提交
967
      if ((argc == i+1) ||
968
        (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
969 970 971 972
        printHelp();
        errorPrint("%s", "\n\t-m need a number following!\n");
        exit(EXIT_FAILURE);
      }
973
      arguments->tb_prefix = argv[++i];
974 975
    } else if (strcmp(argv[i], "-N") == 0) {
      arguments->use_metric = false;
976
    } else if (strcmp(argv[i], "-x") == 0) {
977
      arguments->insert_only = false;
978
    } else if (strcmp(argv[i], "-y") == 0) {
979
      arguments->answer_yes = true;
980 981
    } else if (strcmp(argv[i], "-g") == 0) {
      arguments->debug_print = true;
982 983
    } else if (strcmp(argv[i], "-gg") == 0) {
      arguments->verbose_print = true;
984 985
    } else if (strcmp(argv[i], "-pp") == 0) {
      arguments->performance_print = true;
986
    } else if (strcmp(argv[i], "-O") == 0) {
987 988 989 990 991 992
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-O need a number following!\n");
        exit(EXIT_FAILURE);
      }
993

994
      arguments->disorderRatio = atoi(argv[++i]);
995

996
      if (arguments->disorderRatio > 50) {
997
        arguments->disorderRatio = 50;
998
      }
999

1000
      if (arguments->disorderRatio < 0) {
1001
        arguments->disorderRatio = 0;
1002
      }
1003

1004
    } else if (strcmp(argv[i], "-R") == 0) {
1005 1006 1007 1008 1009 1010
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-R need a number following!\n");
        exit(EXIT_FAILURE);
      }
1011

1012
      arguments->disorderRange = atoi(argv[++i]);
1013 1014 1015
      if (arguments->disorderRange < 0)
        arguments->disorderRange = 1000;

1016
    } else if (strcmp(argv[i], "-a") == 0) {
1017 1018 1019 1020 1021 1022
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-a need a number following!\n");
        exit(EXIT_FAILURE);
      }
1023 1024 1025
      arguments->replica = atoi(argv[++i]);
      if (arguments->replica > 3 || arguments->replica < 1) {
          arguments->replica = 1;
1026
      }
1027 1028
    } else if (strcmp(argv[i], "-D") == 0) {
      arguments->method_of_delete = atoi(argv[++i]);
1029 1030 1031
      if (arguments->method_of_delete > 3) {
        errorPrint("%s", "\n\t-D need a valud (0~3) number following!\n");
        exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1032
      }
1033 1034 1035 1036
    } else if ((strcmp(argv[i], "--version") == 0) ||
        (strcmp(argv[i], "-V") == 0)){
      printVersion();
      exit(0);
1037 1038 1039 1040 1041
    } else if (strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else {
      printHelp();
1042
      errorPrint("%s", "ERROR: wrong options\n");
1043
      exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1044 1045
    }
  }
1046

1047
  if (((arguments->debug_print) && (arguments->metaFile == NULL))
1048
          || arguments->verbose_print) {
1049
    printf("###################################################################\n");
1050
    printf("# meta file:                         %s\n", arguments->metaFile);
1051
    printf("# Server IP:                         %s:%hu\n",
1052 1053
            arguments->host == NULL ? "localhost" : arguments->host,
            arguments->port );
1054 1055
    printf("# User:                              %s\n", arguments->user);
    printf("# Password:                          %s\n", arguments->password);
1056 1057
    printf("# Use metric:                        %s\n",
            arguments->use_metric ? "true" : "false");
1058
    if (*(arguments->datatype)) {
1059 1060 1061 1062 1063 1064 1065
        printf("# Specified data type:               ");
        for (int i = 0; i < MAX_NUM_DATATYPE; i++)
            if (arguments->datatype[i])
               printf("%s,", arguments->datatype[i]);
            else
                break;
        printf("\n");
1066
    }
1067
    printf("# Insertion interval:                %"PRIu64"\n",
1068
            arguments->insert_interval);
1069
    printf("# Number of records per req:         %"PRIu64"\n",
1070
            arguments->num_of_RPR);
1071
    printf("# Max SQL length:                    %"PRIu64"\n",
1072
            arguments->max_sql_len);
1073
    printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
1074
    printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
1075
    printf("# Number of Tables:                  %"PRId64"\n",
1076
            arguments->num_of_tables);
1077
    printf("# Number of Data per Table:          %"PRId64"\n",
1078
            arguments->num_of_DPT);
1079 1080 1081 1082 1083
    printf("# Database name:                     %s\n", arguments->database);
    printf("# Table prefix:                      %s\n", arguments->tb_prefix);
    if (arguments->disorderRatio) {
      printf("# Data order:                        %d\n", arguments->disorderRatio);
      printf("# Data out of order rate:            %d\n", arguments->disorderRange);
1084

1085 1086 1087 1088
    }
    printf("# Delete method:                     %d\n", arguments->method_of_delete);
    printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
    printf("# Print debug info:                  %d\n", arguments->debug_print);
1089
    printf("# Print verbose info:                %d\n", arguments->verbose_print);
1090
    printf("###################################################################\n");
1091 1092

    prompt();
1093
  }
1094
}
H
hzcheng 已提交
1095

1096 1097
static bool getInfoFromJsonFile(char* file);
static void init_rand_data();
1098
static void tmfclose(FILE *fp) {
1099 1100 1101 1102
  if (NULL != fp) {
    fclose(fp);
  }
}
1103

1104
static void tmfree(char *buf) {
1105 1106
  if (NULL != buf) {
    free(buf);
H
Hui Li 已提交
1107
  }
1108
}
H
Hui Li 已提交
1109

1110
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
1111 1112 1113
  int i;
  TAOS_RES *res = NULL;
  int32_t   code = -1;
1114

S
Shuduo Sang 已提交
1115
  for (i = 0; i < 5 /* retry */; i++) {
1116 1117 1118
    if (NULL != res) {
      taos_free_result(res);
      res = NULL;
H
hzcheng 已提交
1119
    }
1120

1121 1122 1123 1124
    res = taos_query(taos, command);
    code = taos_errno(res);
    if (0 == code) {
      break;
1125
    }
H
hzcheng 已提交
1126 1127
  }

1128
  if (code != 0) {
1129 1130
    if (!quiet) {
      debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
1131
      errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res));
1132
    }
1133 1134 1135
    taos_free_result(res);
    //taos_close(taos);
    return -1;
1136
  }
H
hzcheng 已提交
1137

1138 1139 1140 1141
  if (INSERT_TYPE == type) {
    int affectedRows = taos_affected_rows(res);
    taos_free_result(res);
    return affectedRows;
H
hzcheng 已提交
1142
  }
1143

1144
  taos_free_result(res);
1145 1146
  return 0;
}
H
hzcheng 已提交
1147

1148 1149
static void appendResultBufToFile(char *resultBuf, char *resultFile)
{
1150
  FILE *fp = NULL;
1151 1152
  if (resultFile[0] != 0) {
    fp = fopen(resultFile, "at");
1153
    if (fp == NULL) {
1154 1155 1156 1157
      errorPrint(
              "%s() LN%d, failed to open result file: %s, result will not save to file\n",
              __func__, __LINE__, resultFile);
      return;
1158 1159
    }
  }
1160

1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
  fprintf(fp, "%s", resultBuf);
  tmfclose(fp);
}

static void appendResultToFile(TAOS_RES *res, char* resultFile) {
  TAOS_ROW    row = NULL;
  int         num_rows = 0;
  int         num_fields = taos_field_count(res);
  TAOS_FIELD *fields     = taos_fetch_fields(res);

1171 1172
  char* databuf = (char*) calloc(1, 100*1024*1024);
  if (databuf == NULL) {
1173 1174
    errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n",
            __func__, __LINE__);
1175 1176
    return ;
  }
H
hzcheng 已提交
1177

1178 1179
  int   totalLen = 0;
  char  temp[16000];
H
hzcheng 已提交
1180

1181
  // fetch the records row by row
1182
  while((row = taos_fetch_row(res))) {
1183
    if (totalLen >= 100*1024*1024 - 32000) {
1184
      appendResultBufToFile(databuf, resultFile);
1185 1186
      totalLen = 0;
      memset(databuf, 0, 100*1024*1024);
H
hzcheng 已提交
1187
    }
1188 1189 1190 1191 1192 1193
    num_rows++;
    int len = taos_print_row(temp, row, fields, num_fields);
    len += sprintf(temp + len, "\n");
    //printf("query result:%s\n", temp);
    memcpy(databuf + totalLen, temp, len);
    totalLen += len;
H
hzcheng 已提交
1194 1195
  }

1196 1197
  verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n",
          __func__, __LINE__, databuf, resultFile);
1198
  appendResultBufToFile(databuf, resultFile);
1199 1200
  free(databuf);
}
H
hzcheng 已提交
1201

1202
static void selectAndGetResult(
1203 1204
        threadInfo *pThreadInfo, char *command)
{
1205 1206 1207 1208 1209 1210 1211 1212 1213
  if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) {
    TAOS_RES *res = taos_query(pThreadInfo->taos, command);
    if (res == NULL || taos_errno(res) != 0) {
        errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n",
            __func__, __LINE__, command, taos_errstr(res));
        taos_free_result(res);
        return;
    }

1214 1215
    if ((strlen(pThreadInfo->fp))) {
      appendResultToFile(res, pThreadInfo->fp);
1216
    }
1217
    taos_free_result(res);
1218

1219 1220 1221
  } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
      int retCode = postProceSql(
              g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port,
1222
              command,
1223
              pThreadInfo->fp);
1224 1225 1226 1227 1228 1229 1230 1231
      if (0 != retCode) {
        printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID);
      }

  } else {
      errorPrint("%s() LN%d, unknown query mode: %s\n",
        __func__, __LINE__, g_queryInfo.queryMode);
  }
1232
}
H
hzcheng 已提交
1233

1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272
static int32_t rand_bool(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 2;
}

static int32_t rand_tinyint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 128;
}

static int32_t rand_smallint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 32767;
}

static int32_t rand_int(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor];
}

static int64_t rand_bigint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randbigint[cursor];
}

static float rand_float(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
1273
  return randfloat[cursor];
1274 1275
}

1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
#if 0
static const char charNum[] = "0123456789";

static void nonrand_string(char *, int) __attribute__ ((unused));   // reserve for debugging purpose
static void nonrand_string(char *str, int size)
{
  str[0] = 0;
  if (size > 0) {
    int n;
    for (n = 0; n < size; n++) {
      str[n] = charNum[n % 10];
    }
    str[n] = 0;
  }
}
#endif

1293
static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1294

1295
static void rand_string(char *str, int size) {
1296 1297 1298 1299
  str[0] = 0;
  if (size > 0) {
    //--size;
    int n;
1300
    for (n = 0; n < size; n++) {
1301
      int key = abs(rand_tinyint()) % (int)(sizeof(charset) - 1);
1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317
      str[n] = charset[key];
    }
    str[n] = 0;
  }
}

static double rand_double() {
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randdouble[cursor];

}

static void init_rand_data() {
  for (int i = 0; i < MAX_PREPARED_RAND; i++){
1318 1319 1320 1321
    randint[i] = (int)(taosRandom() % 65535);
    randbigint[i] = (int64_t)(taosRandom() % 2147483648);
    randfloat[i] = (float)(taosRandom() / 1000.0);
    randdouble[i] = (double)(taosRandom() / 1000000.0);
1322 1323 1324
  }
}

1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1345
static int printfInsertMeta() {
1346
    SHOW_PARSE_RESULT_START();
1347

1348
  printf("interface:                  \033[33m%s\033[0m\n",
S
Shuduo Sang 已提交
1349
          (g_args.iface==TAOSC_IFACE)?"taosc":(g_args.iface==REST_IFACE)?"rest":"stmt");
1350 1351
  printf("host:                       \033[33m%s:%u\033[0m\n",
          g_Dbs.host, g_Dbs.port);
1352 1353
  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
1354
  printf("configDir:                  \033[33m%s\033[0m\n", configDir);
1355 1356
  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
1357 1358
  printf("thread num of create table: \033[33m%d\033[0m\n",
          g_Dbs.threadCountByCreateTbl);
1359
  printf("top insert interval:        \033[33m%"PRIu64"\033[0m\n",
1360
          g_args.insert_interval);
1361
  printf("number of records per req:  \033[33m%"PRIu64"\033[0m\n",
1362
          g_args.num_of_RPR);
1363
  printf("max sql length:             \033[33m%"PRIu64"\033[0m\n",
1364
          g_args.max_sql_len);
1365 1366

  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
1367

1368 1369
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    printf("database[\033[33m%d\033[0m]:\n", i);
1370 1371
    printf("  database[%d] name:      \033[33m%s\033[0m\n",
            i, g_Dbs.db[i].dbName);
1372
    if (0 == g_Dbs.db[i].drop) {
1373 1374 1375
      printf("  drop:                  \033[33mno\033[0m\n");
    } else {
      printf("  drop:                  \033[33myes\033[0m\n");
1376 1377 1378
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
1379 1380
      printf("  blocks:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.blocks);
1381 1382
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
1383 1384
      printf("  cache:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.cache);
1385 1386
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
1387 1388
      printf("  days:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.days);
1389 1390
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
1391 1392
      printf("  keep:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.keep);
1393 1394
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
1395 1396
      printf("  replica:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.replica);
1397 1398
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
1399 1400
      printf("  update:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.update);
1401 1402
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
1403 1404
      printf("  minRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.minRows);
1405 1406
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
1407 1408
      printf("  maxRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.maxRows);
1409 1410 1411 1412 1413
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
1414 1415
      printf("  walLevel:              \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.walLevel);
1416 1417
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
1418 1419
      printf("  fsync:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.fsync);
1420 1421
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
1422 1423
      printf("  quorum:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.quorum);
1424 1425
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1426
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
1427
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1428
        printf("  precision:             \033[33m%s\033[0m\n",
1429
            g_Dbs.db[i].dbCfg.precision);
1430
      } else {
1431
        printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
1432
                g_Dbs.db[i].dbCfg.precision);
1433
        return -1;
1434 1435 1436
      }
    }

1437
    printf("  super table count:     \033[33m%"PRIu64"\033[0m\n",
1438
        g_Dbs.db[i].superTblCount);
1439 1440
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      printf("  super table[\033[33m%"PRIu64"\033[0m]:\n", j);
1441

1442
      printf("      stbName:           \033[33m%s\033[0m\n",
1443
          g_Dbs.db[i].superTbls[j].sTblName);
1444 1445 1446 1447 1448 1449 1450 1451

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
      }
1452

1453 1454 1455 1456 1457 1458 1459
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
      }
1460

1461
      printf("      childTblCount:     \033[33m%"PRId64"\033[0m\n",
1462
              g_Dbs.db[i].superTbls[j].childTblCount);
1463
      printf("      childTblPrefix:    \033[33m%s\033[0m\n",
1464
              g_Dbs.db[i].superTbls[j].childTblPrefix);
1465
      printf("      dataSource:        \033[33m%s\033[0m\n",
1466
              g_Dbs.db[i].superTbls[j].dataSource);
1467
      printf("      insertMode:        \033[33m%s\033[0m\n",
1468 1469
          (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc":
          (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt");
1470
      if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
1471
        printf("      childTblLimit:     \033[33m%"PRId64"\033[0m\n",
1472
                g_Dbs.db[i].superTbls[j].childTblLimit);
1473
      }
1474
      if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) {
1475
        printf("      childTblOffset:    \033[33m%"PRIu64"\033[0m\n",
1476
                g_Dbs.db[i].superTbls[j].childTblOffset);
1477
      }
1478
      printf("      insertRows:        \033[33m%"PRId64"\033[0m\n",
1479
              g_Dbs.db[i].superTbls[j].insertRows);
1480
/*
1481
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1482
        printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
1483
      }else {
1484
        printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
1485
      }
1486
      */
1487
      printf("      interlaceRows:     \033[33m%"PRIu64"\033[0m\n",
1488
              g_Dbs.db[i].superTbls[j].interlaceRows);
1489 1490

      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1491
        printf("      stable insert interval:   \033[33m%"PRIu64"\033[0m\n",
1492
            g_Dbs.db[i].superTbls[j].insertInterval);
1493 1494
      }

1495
      printf("      disorderRange:     \033[33m%d\033[0m\n",
1496
              g_Dbs.db[i].superTbls[j].disorderRange);
1497
      printf("      disorderRatio:     \033[33m%d\033[0m\n",
1498
              g_Dbs.db[i].superTbls[j].disorderRatio);
1499
      printf("      maxSqlLen:         \033[33m%"PRIu64"\033[0m\n",
1500
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1501
      printf("      timeStampStep:     \033[33m%"PRId64"\033[0m\n",
1502
              g_Dbs.db[i].superTbls[j].timeStampStep);
1503
      printf("      startTimestamp:    \033[33m%s\033[0m\n",
1504
              g_Dbs.db[i].superTbls[j].startTimestamp);
1505
      printf("      sampleFormat:      \033[33m%s\033[0m\n",
1506
              g_Dbs.db[i].superTbls[j].sampleFormat);
1507
      printf("      sampleFile:        \033[33m%s\033[0m\n",
1508
              g_Dbs.db[i].superTbls[j].sampleFile);
1509
      printf("      tagsFile:          \033[33m%s\033[0m\n",
1510
              g_Dbs.db[i].superTbls[j].tagsFile);
1511
      printf("      columnCount:       \033[33m%d\033[0m\n",
1512
              g_Dbs.db[i].superTbls[j].columnCount);
1513 1514
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1515
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
1516 1517 1518
                       "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "nchar", 5))) {
1519
          printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
1520 1521
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1522
        } else {
1523
          printf("column[%d]:\033[33m%s\033[0m ", k,
1524
                  g_Dbs.db[i].superTbls[j].columns[k].dataType);
1525 1526 1527
        }
      }
      printf("\n");
1528 1529

      printf("      tagCount:            \033[33m%d\033[0m\n        ",
1530
              g_Dbs.db[i].superTbls[j].tagCount);
1531 1532
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1533
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
1534 1535 1536
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1537
          printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
1538 1539
                  g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1540
        } else {
1541
          printf("tag[%d]:\033[33m%s\033[0m ", k,
1542
                  g_Dbs.db[i].superTbls[j].tags[k].dataType);
1543
        }
1544 1545 1546 1547 1548
      }
      printf("\n");
    }
    printf("\n");
  }
1549 1550

  SHOW_PARSE_RESULT_END();
1551 1552

  return 0;
1553 1554 1555
}

static void printfInsertMetaToFile(FILE* fp) {
1556 1557

  SHOW_PARSE_RESULT_START_TO_FILE(fp);
1558

1559 1560
  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
  fprintf(fp, "user:                       %s\n", g_Dbs.user);
1561
  fprintf(fp, "configDir:                  %s\n", configDir);
1562 1563 1564
  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
1565 1566
  fprintf(fp, "number of records per req:  %"PRIu64"\n", g_args.num_of_RPR);
  fprintf(fp, "max sql length:             %"PRIu64"\n", g_args.max_sql_len);
1567
  fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
1568

1569 1570
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    fprintf(fp, "database[%d]:\n", i);
1571
    fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
1572
    if (0 == g_Dbs.db[i].drop) {
1573
      fprintf(fp, "  drop:                  no\n");
1574
    }else {
1575
      fprintf(fp, "  drop:                  yes\n");
1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1615
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
1616
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1617 1618
        fprintf(fp, "  precision:             %s\n",
                g_Dbs.db[i].dbCfg.precision);
1619
      } else {
1620 1621
        fprintf(fp, "  precision error:       %s\n",
                g_Dbs.db[i].dbCfg.precision);
1622 1623 1624
      }
    }

1625 1626
    fprintf(fp, "  super table count:     %"PRIu64"\n",
            g_Dbs.db[i].superTblCount);
1627 1628
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      fprintf(fp, "  super table[%d]:\n", j);
1629

1630 1631
      fprintf(fp, "      stbName:           %s\n",
              g_Dbs.db[i].superTbls[j].sTblName);
1632 1633 1634

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "no");
1635 1636
      } else if (AUTO_CREATE_SUBTBL
              == g_Dbs.db[i].superTbls[j].autoCreateTable) {
1637 1638 1639 1640
        fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
      } else {
        fprintf(fp, "      autoCreateTable:   %s\n",  "error");
      }
1641

1642 1643
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "no");
1644 1645
      } else if (TBL_ALREADY_EXISTS
              == g_Dbs.db[i].superTbls[j].childTblExists) {
1646 1647 1648 1649
        fprintf(fp, "      childTblExists:    %s\n",  "yes");
      } else {
        fprintf(fp, "      childTblExists:    %s\n",  "error");
      }
1650

1651
      fprintf(fp, "      childTblCount:     %"PRId64"\n",
1652 1653 1654 1655 1656 1657
              g_Dbs.db[i].superTbls[j].childTblCount);
      fprintf(fp, "      childTblPrefix:    %s\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      fprintf(fp, "      dataSource:        %s\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      fprintf(fp, "      insertMode:        %s\n",
1658 1659
          (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc":
          (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt");
1660
      fprintf(fp, "      insertRows:        %"PRId64"\n",
1661
              g_Dbs.db[i].superTbls[j].insertRows);
1662
      fprintf(fp, "      interlace rows:    %"PRIu64"\n",
1663
              g_Dbs.db[i].superTbls[j].interlaceRows);
1664
      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1665
        fprintf(fp, "      stable insert interval:   %"PRIu64"\n",
1666
                g_Dbs.db[i].superTbls[j].insertInterval);
1667
      }
1668
/*
1669
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1670
        fprintf(fp, "      multiThreadWriteOneTbl:  no\n");
1671
      }else {
1672
        fprintf(fp, "      multiThreadWriteOneTbl:  yes\n");
1673
      }
1674
      */
1675
      fprintf(fp, "      interlaceRows:     %"PRIu64"\n",
1676
              g_Dbs.db[i].superTbls[j].interlaceRows);
1677 1678 1679 1680
      fprintf(fp, "      disorderRange:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      fprintf(fp, "      disorderRatio:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
1681
      fprintf(fp, "      maxSqlLen:         %"PRIu64"\n",
1682
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1683

1684 1685 1686 1687
      fprintf(fp, "      timeStampStep:     %"PRId64"\n",
              g_Dbs.db[i].superTbls[j].timeStampStep);
      fprintf(fp, "      startTimestamp:    %s\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
1688 1689 1690 1691 1692 1693
      fprintf(fp, "      sampleFormat:      %s\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      fprintf(fp, "      sampleFile:        %s\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      fprintf(fp, "      tagsFile:          %s\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
1694

1695 1696
      fprintf(fp, "      columnCount:       %d\n        ",
              g_Dbs.db[i].superTbls[j].columnCount);
1697 1698
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1699
        if ((0 == strncasecmp(
1700 1701
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "binary", strlen("binary")))
1702 1703
                || (0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
1704
                        "nchar", strlen("nchar")))) {
1705 1706 1707
          fprintf(fp, "column[%d]:%s(%d) ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1708
        } else {
1709 1710
          fprintf(fp, "column[%d]:%s ",
                  k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
1711 1712 1713
        }
      }
      fprintf(fp, "\n");
1714 1715 1716

      fprintf(fp, "      tagCount:            %d\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1717 1718
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1719
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
1720 1721 1722
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1723 1724
          fprintf(fp, "tag[%d]:%s(%d) ",
                  k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
1725
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1726 1727
        } else {
          fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
1728
        }
1729 1730 1731 1732 1733
      }
      fprintf(fp, "\n");
    }
    fprintf(fp, "\n");
  }
1734

1735
  SHOW_PARSE_RESULT_END_TO_FILE(fp);
1736 1737 1738
}

static void printfQueryMeta() {
1739

1740
  SHOW_PARSE_RESULT_START();
1741

1742
  printf("host:                    \033[33m%s:%u\033[0m\n",
1743
          g_queryInfo.host, g_queryInfo.port);
1744 1745 1746 1747
  printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
  printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

  printf("\n");
1748 1749 1750 1751

  if ((SUBSCRIBE_TEST == g_args.test_mode) || (QUERY_TEST == g_args.test_mode)) {
    printf("specified table query info:                   \n");
    printf("sqlCount:       \033[33m%"PRIu64"\033[0m\n",
1752
      g_queryInfo.specifiedQueryInfo.sqlCount);
1753 1754 1755
    if (g_queryInfo.specifiedQueryInfo.sqlCount > 0) {
      printf("specified tbl query times:\n");
      printf("                \033[33m%"PRIu64"\033[0m\n",
1756
      g_queryInfo.specifiedQueryInfo.queryTimes);
1757 1758 1759 1760 1761 1762
      printf("query interval: \033[33m%"PRIu64" ms\033[0m\n",
        g_queryInfo.specifiedQueryInfo.queryInterval);
      printf("top query times:\033[33m%"PRIu64"\033[0m\n", g_args.query_times);
      printf("concurrent:     \033[33m%"PRIu64"\033[0m\n",
      g_queryInfo.specifiedQueryInfo.concurrent);
      printf("mod:            \033[33m%s\033[0m\n",
1763
        (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync");
1764
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1765
        g_queryInfo.specifiedQueryInfo.subscribeInterval);
1766
      printf("restart:        \033[33m%d\033[0m\n",
1767
        g_queryInfo.specifiedQueryInfo.subscribeRestart);
1768
      printf("keepProgress:   \033[33m%d\033[0m\n",
1769
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
1770

1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786
      for (uint64_t i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        printf("  sql[%"PRIu64"]: \033[33m%s\033[0m\n",
            i, g_queryInfo.specifiedQueryInfo.sql[i]);
      }
      printf("\n");
    }

    printf("super table query info:\n");
    printf("sqlCount:       \033[33m%"PRIu64"\033[0m\n",
      g_queryInfo.superQueryInfo.sqlCount);

    if (g_queryInfo.superQueryInfo.sqlCount > 0) {
      printf("query interval: \033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryInterval);
      printf("threadCnt:      \033[33m%d\033[0m\n",
        g_queryInfo.superQueryInfo.threadCnt);
1787
      printf("childTblCount:  \033[33m%"PRId64"\033[0m\n",
1788 1789 1790 1791 1792 1793 1794
        g_queryInfo.superQueryInfo.childTblCount);
      printf("stable name:    \033[33m%s\033[0m\n",
        g_queryInfo.superQueryInfo.sTblName);
      printf("stb query times:\033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryTimes);

      printf("mod:            \033[33m%s\033[0m\n",
1795
        (g_queryInfo.superQueryInfo.asyncMode)?"async":"sync");
1796
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1797
        g_queryInfo.superQueryInfo.subscribeInterval);
1798
      printf("restart:        \033[33m%d\033[0m\n",
1799
        g_queryInfo.superQueryInfo.subscribeRestart);
1800
      printf("keepProgress:   \033[33m%d\033[0m\n",
1801
        g_queryInfo.superQueryInfo.subscribeKeepProgress);
1802

1803 1804 1805 1806 1807 1808
      for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
        printf("  sql[%d]: \033[33m%s\033[0m\n",
            i, g_queryInfo.superQueryInfo.sql[i]);
      }
      printf("\n");
    }
1809
  }
1810

1811
  SHOW_PARSE_RESULT_END();
1812 1813
}

1814
static char* formatTimestamp(char* buf, int64_t val, int precision) {
1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845
  time_t tt;
  if (precision == TSDB_TIME_PRECISION_MICRO) {
    tt = (time_t)(val / 1000000);
  } else {
    tt = (time_t)(val / 1000);
  }

/* comment out as it make testcases like select_with_tags.sim fail.
  but in windows, this may cause the call to localtime crash if tt < 0,
  need to find a better solution.
  if (tt < 0) {
    tt = 0;
  }
  */

#ifdef WINDOWS
  if (tt < 0) tt = 0;
#endif

  struct tm* ptm = localtime(&tt);
  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);

  if (precision == TSDB_TIME_PRECISION_MICRO) {
    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
  } else {
    sprintf(buf + pos, ".%03d", (int)(val % 1000));
  }

  return buf;
}

1846
static void xDumpFieldToFile(FILE* fp, const char* val,
1847
        TAOS_FIELD* field, int32_t length, int precision) {
1848

1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883
  if (val == NULL) {
    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
    return;
  }

  char buf[TSDB_MAX_BYTES_PER_ROW];
  switch (field->type) {
    case TSDB_DATA_TYPE_BOOL:
      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
      break;
    case TSDB_DATA_TYPE_TINYINT:
      fprintf(fp, "%d", *((int8_t *)val));
      break;
    case TSDB_DATA_TYPE_SMALLINT:
      fprintf(fp, "%d", *((int16_t *)val));
      break;
    case TSDB_DATA_TYPE_INT:
      fprintf(fp, "%d", *((int32_t *)val));
      break;
    case TSDB_DATA_TYPE_BIGINT:
      fprintf(fp, "%" PRId64, *((int64_t *)val));
      break;
    case TSDB_DATA_TYPE_FLOAT:
      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
      break;
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      memcpy(buf, val, length);
      buf[length] = 0;
      fprintf(fp, "\'%s\'", buf);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
1884
      formatTimestamp(buf, *(int64_t*)val, precision);
1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899
      fprintf(fp, "'%s'", buf);
      break;
    default:
      break;
  }
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  FILE* fp = fopen(fname, "at");
  if (fp == NULL) {
1900
    errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914
    return -1;
  }

  int num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int precision = taos_result_precision(tres);

  for (int col = 0; col < num_fields; col++) {
    if (col > 0) {
      fprintf(fp, ",");
    }
    fprintf(fp, "%s", fields[col].name);
  }
  fputc('\n', fp);
1915

1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935
  int numOfRows = 0;
  do {
    int32_t* length = taos_fetch_lengths(tres);
    for (int i = 0; i < num_fields; i++) {
      if (i > 0) {
        fputc(',', fp);
      }
      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
    }
    fputc('\n', fp);

    numOfRows++;
    row = taos_fetch_row(tres);
  } while( row != NULL);

  fclose(fp);

  return numOfRows;
}

1936 1937
static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
  TAOS_RES * res;
1938 1939
  TAOS_ROW row = NULL;
  int count = 0;
1940 1941

  res = taos_query(taos, "show databases;");
1942
  int32_t code = taos_errno(res);
1943

1944
  if (code != 0) {
1945
    errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
1946 1947 1948 1949 1950
    return -1;
  }

  TAOS_FIELD *fields = taos_fetch_fields(res);

1951
  while((row = taos_fetch_row(res)) != NULL) {
1952
    // sys database name : 'log'
1953
    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log",
1954
                fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) {
1955 1956
      continue;
    }
1957 1958 1959

    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
    if (dbInfos[count] == NULL) {
1960
      errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
1961 1962 1963
      return -1;
    }

1964
    tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
1965
            fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
1966
    formatTimestamp(dbInfos[count]->create_time,
1967 1968
            *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
            TSDB_TIME_PRECISION_MILLI);
S
Shuduo Sang 已提交
1969
    dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
1970
    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
1971 1972
    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
1973
    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
1974

1975
    tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
1976
            fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
1977 1978 1979 1980 1981 1982 1983
    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
1984
    dbInfos[count]->cachelast =
1985
      (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));
1986

1987
    tstrncpy(dbInfos[count]->precision,
1988 1989
            (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
            fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
1990
    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
1991
    tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
1992
            fields[TSDB_SHOW_DB_STATUS_INDEX].bytes);
1993

1994 1995
    count++;
    if (count > MAX_DATABASE_COUNT) {
1996
      errorPrint("%s() LN%d, The database count overflow than %d\n",
1997
         __func__, __LINE__, MAX_DATABASE_COUNT);
1998 1999 2000 2001 2002 2003 2004
      break;
    }
  }

  return count;
}

2005
static void printfDbInfoForQueryToFile(
2006
        char* filename, SDbInfo* dbInfos, int index) {
2007

2008
  if (filename[0] == 0)
2009
      return;
2010 2011 2012

  FILE *fp = fopen(filename, "at");
  if (fp == NULL) {
2013
    errorPrint( "failed to open file: %s\n", filename);
2014
    return;
2015 2016 2017 2018 2019
  }

  fprintf(fp, "================ database[%d] ================\n", index);
  fprintf(fp, "name: %s\n", dbInfos->name);
  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
S
Shuduo Sang 已提交
2020
  fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables);
2021
  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);
2022 2023
  fprintf(fp, "replica: %d\n", dbInfos->replica);
  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
2024 2025
  fprintf(fp, "days: %d\n", dbInfos->days);
  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);
2026 2027 2028 2029 2030 2031 2032
  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
  fprintf(fp, "comp: %d\n", dbInfos->comp);
2033 2034
  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);
  fprintf(fp, "precision: %s\n", dbInfos->precision);
2035
  fprintf(fp, "update: %d\n", dbInfos->update);
2036
  fprintf(fp, "status: %s\n", dbInfos->status);
2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050
  fprintf(fp, "\n");

  fclose(fp);
}

static void printfQuerySystemInfo(TAOS * taos) {
  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
  TAOS_RES* res;

  time_t t;
  struct tm* lt;
  time(&t);
  lt = localtime(&t);
2051
  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
2052 2053
          lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
          lt->tm_sec);
2054

2055 2056
  // show variables
  res = taos_query(taos, "show variables;");
2057
  //appendResultToFile(res, filename);
2058 2059 2060 2061 2062
  xDumpResultToFile(filename, res);

  // show dnodes
  res = taos_query(taos, "show dnodes;");
  xDumpResultToFile(filename, res);
2063
  //appendResultToFile(res, filename);
2064

2065 2066 2067 2068
  // show databases
  res = taos_query(taos, "show databases;");
  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
  if (dbInfos == NULL) {
2069
    errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
2070 2071 2072
    return;
  }
  int dbCount = getDbFromServer(taos, dbInfos);
2073
  if (dbCount <= 0) {
2074 2075
      free(dbInfos);
      return;
2076
  }
2077 2078

  for (int i = 0; i < dbCount; i++) {
2079
    // printf database info
2080
    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
2081

2082 2083 2084 2085
    // show db.vgroups
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);
2086

2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097
    // show db.stables
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);

    free(dbInfos[i]);
  }

  free(dbInfos);
}

2098 2099
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
        char* sqlstr, char *resultFile)
2100
{
2101
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
2102

2103
    char *url = "/rest/sql";
2104

2105 2106 2107 2108
    int bytes, sent, received, req_str_len, resp_len;
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
2109

2110
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;
2111

2112 2113 2114 2115 2116
    request_buf = malloc(req_buf_len);
    if (NULL == request_buf) {
      errorPrint("%s", "ERROR, cannot allocate memory.\n");
      exit(EXIT_FAILURE);
    }
2117

2118 2119
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};
2120

2121 2122 2123 2124 2125 2126 2127 2128
    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
2129

2130 2131 2132 2133
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);
2134

2135
    char base64_buf[INPUT_BUF_LEN];
2136
#ifdef WINDOWS
2137
    WSADATA wsaData;
2138
    WSAStartup(MAKEWORD(2, 2), &wsaData);
2139 2140
    SOCKET sockfd;
#else
2141
    int sockfd;
2142
#endif
2143 2144
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (sockfd < 0) {
2145
#ifdef WINDOWS
2146
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
2147
#endif
2148 2149 2150 2151
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
        free(request_buf);
        ERROR_EXIT("ERROR opening socket");
    }
2152

2153
    int retConn = connect(sockfd, (struct sockaddr *)pServAddr, sizeof(struct sockaddr));
2154 2155 2156 2157 2158
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
        free(request_buf);
        ERROR_EXIT("ERROR connecting");
    }
2159

2160
    memset(base64_buf, 0, INPUT_BUF_LEN);
2161

2162 2163 2164 2165 2166 2167 2168 2169
    for (int n = 0, m = 0; n < userpass_buf_len;) {
      uint32_t oct_a = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t oct_b = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t oct_c = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;
2170

2171 2172 2173 2174 2175
      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }
2176

2177 2178
    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';
2179

2180 2181 2182
    debugPrint("%s() LN%d: auth string base64 encoded: %s\n",
            __func__, __LINE__, base64_buf);
    char *auth = base64_buf;
2183

2184 2185 2186 2187 2188 2189 2190 2191 2192
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
2193

2194 2195 2196
    req_str_len = strlen(request_buf);
    sent = 0;
    do {
2197
#ifdef WINDOWS
2198
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
2199
#else
2200
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
2201
#endif
2202 2203 2204 2205 2206 2207
        if (bytes < 0)
            ERROR_EXIT("ERROR writing message to socket");
        if (bytes == 0)
            break;
        sent+=bytes;
    } while(sent < req_str_len);
2208

2209 2210 2211 2212
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
    received = 0;
    do {
2213
#ifdef WINDOWS
2214
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
2215
#else
2216
        bytes = read(sockfd, response_buf + received, resp_len - received);
2217
#endif
2218 2219 2220 2221 2222 2223 2224 2225
        if (bytes < 0) {
            free(request_buf);
            ERROR_EXIT("ERROR reading response from socket");
        }
        if (bytes == 0)
            break;
        received += bytes;
    } while(received < resp_len);
2226

2227 2228 2229 2230
    if (received == resp_len) {
        free(request_buf);
        ERROR_EXIT("ERROR storing complete response from socket");
    }
H
Haojun Liao 已提交
2231

2232 2233
    response_buf[RESP_BUF_LEN - 1] = '\0';
    printf("Response:\n%s\n", response_buf);
H
Haojun Liao 已提交
2234

2235 2236 2237 2238
    if (resultFile) {
       appendResultBufToFile(response_buf, resultFile);
    }

2239
    free(request_buf);
2240
#ifdef WINDOWS
2241
    closesocket(sockfd);
2242 2243
    WSACleanup();
#else
2244
    close(sockfd);
2245
#endif
2246

2247
    return 0;
2248 2249
}

2250
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
2251 2252
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
2253
    errorPrint("%s() LN%d, calloc failed! size:%d\n",
2254
        __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
2255 2256
    return NULL;
  }
2257

2258
  int    dataLen = 0;
2259
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2260
          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);
2261

2262 2263 2264
  return dataBuf;
}

2265
static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) {
2266 2267 2268 2269 2270 2271 2272 2273 2274
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
    printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
    return NULL;
  }

  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
  for (int i = 0; i < stbInfo->tagCount; i++) {
2275
    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
2276
            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
2277
      if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
2278
        printf("binary or nchar length overflow, max size:%u\n",
2279
                (uint32_t)TSDB_MAX_BINARY_LEN);
2280 2281 2282
        tmfree(dataBuf);
        return NULL;
      }
2283

2284 2285
      int tagBufLen = stbInfo->tags[i].dataLen + 1;
      char* buf = (char*)calloc(tagBufLen, 1);
2286 2287 2288 2289 2290
      if (NULL == buf) {
        printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
        tmfree(dataBuf);
        return NULL;
      }
2291 2292 2293 2294 2295 2296 2297

      if (tableSeq % 2) {
        tstrncpy(buf, "beijing", tagBufLen);
      } else {
        tstrncpy(buf, "shanghai", tagBufLen);
      }
      //rand_string(buf, stbInfo->tags[i].dataLen);
2298
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2299
              "\'%s\', ", buf);
2300
      tmfree(buf);
2301
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2302
                "int", strlen("int"))) {
2303
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2304
              "%d, ", tableSeq);
2305
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2306
                "bigint", strlen("bigint"))) {
2307
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2308
              "%"PRId64", ", rand_bigint());
2309
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2310
                "float", strlen("float"))) {
2311
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2312
              "%f, ", rand_float());
2313
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2314
                "double", strlen("double"))) {
2315
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2316
              "%f, ", rand_double());
2317
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2318
                "smallint", strlen("smallint"))) {
2319
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2320
              "%d, ", rand_smallint());
2321
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2322
                "tinyint", strlen("tinyint"))) {
2323
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2324
              "%d, ", rand_tinyint());
2325
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2326
                "bool", strlen("bool"))) {
2327
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2328
              "%d, ", rand_bool());
2329
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2330
                "timestamp", strlen("timestamp"))) {
2331
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2332
              "%"PRId64", ", rand_bigint());
2333 2334 2335 2336 2337 2338
    }  else {
      printf("No support data type: %s\n", stbInfo->tags[i].dataType);
      tmfree(dataBuf);
      return NULL;
    }
  }
2339

2340
  dataLen -= 2;
2341
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");
2342 2343 2344
  return dataBuf;
}

2345
static int calcRowLen(SSuperTable*  superTbls) {
2346 2347
  int colIndex;
  int  lenOfOneRow = 0;
2348

2349 2350
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2351

2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
2368
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2369
      lenOfOneRow += 42;
2370
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
2384

2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
2401
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2402 2403 2404 2405 2406 2407 2408 2409
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2410

2411 2412 2413 2414
  return 0;
}


2415
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
2416
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
2417
        int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) {
2418

2419
  char command[BUFFER_SIZE] = "\0";
2420 2421
  char limitBuf[100] = "\0";

2422
  TAOS_RES * res;
2423 2424 2425
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2426 2427

  if (offset >= 0) {
2428
    snprintf(limitBuf, 100, " limit %"PRId64" offset %"PRIu64"",
2429
            limit, offset);
2430 2431
  }

2432
  //get all child table name use cmd: select tbname from superTblName;
2433
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s",
2434
          dbName, sTblName, limitBuf);
2435

2436
  res = taos_query(taos, command);
2437 2438 2439 2440
  int32_t code = taos_errno(res);
  if (code != 0) {
    taos_free_result(res);
    taos_close(taos);
2441
    errorPrint("%s() LN%d, failed to run command %s\n",
2442
           __func__, __LINE__, command);
2443 2444 2445
    exit(-1);
  }

2446 2447
  int64_t childTblCount = (limit < 0)?10000:limit;
  int64_t count = 0;
2448 2449 2450
  if (childTblName == NULL) {
    childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (NULL ==  childTblName) {
2451 2452 2453 2454
    taos_free_result(res);
        taos_close(taos);
        errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__);
        exit(-1);
2455 2456 2457
    }
  }

2458
  char* pTblName = childTblName;
2459
  while((row = taos_fetch_row(res)) != NULL) {
2460
    int32_t* len = taos_fetch_lengths(res);
2461
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2462 2463 2464
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2465
      char *tmp = realloc(childTblName,
2466
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2467 2468 2469
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2470
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
2471
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2472 2473
      } else {
        // exit, if allocate more memory failed
2474
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
2475
               __func__, __LINE__, dbName, sTblName);
2476 2477 2478 2479 2480 2481
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2482
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2483
  }
2484

2485 2486 2487 2488 2489 2490 2491
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2492
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
2493
        char* sTblName, char** childTblNameOfSuperTbl,
2494
        int64_t* childTblCountOfSuperTbl) {
2495

2496 2497
    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
2498
            -1, 0);
2499 2500
}

2501
static int getSuperTableFromServer(TAOS * taos, char* dbName,
2502
        SSuperTable*  superTbls) {
2503

2504
  char command[BUFFER_SIZE] = "\0";
2505
  TAOS_RES * res;
2506 2507
  TAOS_ROW row = NULL;
  int count = 0;
2508

2509
  //get schema use cmd: describe superTblName;
2510
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2511
  res = taos_query(taos, command);
2512 2513 2514
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2515
    taos_free_result(res);
2516 2517 2518 2519 2520 2521
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
2522
  while((row = taos_fetch_row(res)) != NULL) {
2523 2524 2525
    if (0 == count) {
      count++;
      continue;
2526
    }
2527 2528

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2529
      tstrncpy(superTbls->tags[tagIndex].field,
2530 2531
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
2532
      tstrncpy(superTbls->tags[tagIndex].dataType,
2533 2534
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
2535 2536 2537
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
2538 2539
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2540
      tagIndex++;
2541
    } else {
2542
      tstrncpy(superTbls->columns[columnIndex].field,
2543 2544
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
2545
      tstrncpy(superTbls->columns[columnIndex].dataType,
2546 2547
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
2548 2549 2550
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
2551 2552
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2564
/*
2565
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
2566
    //get all child table name use cmd: select tbname from superTblName;
2567 2568 2569
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2570
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2571 2572
      return -1;
    }
2573
    getAllChildNameOfSuperTable(taos, dbName,
2574 2575 2576
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2577
  }
2578
  */
2579 2580 2581
  return 0;
}

H
Haojun Liao 已提交
2582
static int createSuperTable(
2583 2584
        TAOS * taos, char* dbName,
        SSuperTable*  superTbl) {
H
Haojun Liao 已提交
2585

2586
  char command[BUFFER_SIZE] = "\0";
2587

2588 2589 2590 2591 2592
  char cols[STRING_LEN] = "\0";
  int colIndex;
  int len = 0;

  int  lenOfOneRow = 0;
2593 2594 2595

  if (superTbl->columnCount == 0) {
    errorPrint("%s() LN%d, super table column count is %d\n",
2596
            __func__, __LINE__, superTbl->columnCount);
2597 2598 2599 2600 2601
    return -1;
  }

  for (colIndex = 0; colIndex < superTbl->columnCount; colIndex++) {
    char* dataType = superTbl->columns[colIndex].dataType;
2602

2603
    if (strcasecmp(dataType, "BINARY") == 0) {
2604
      len += snprintf(cols + len, STRING_LEN - len,
2605 2606
          ", col%d %s(%d)", colIndex, "BINARY",
          superTbl->columns[colIndex].dataLen);
2607
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2608
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
2609
      len += snprintf(cols + len, STRING_LEN - len,
2610 2611
          ", col%d %s(%d)", colIndex, "NCHAR",
          superTbl->columns[colIndex].dataLen);
2612
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
      lenOfOneRow += 22;
2631
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2632 2633
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
      lenOfOneRow += 42;
2634
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2635 2636 2637 2638
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
      lenOfOneRow += 21;
    } else {
      taos_close(taos);
2639
      errorPrint("%s() LN%d, config error data type : %s\n",
2640
         __func__, __LINE__, dataType);
2641 2642 2643 2644
      exit(-1);
    }
  }

2645 2646
  superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp
  //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbl[j].sTblName, g_Dbs.db[i].superTbl[j].columnCount, lenOfOneRow);
2647 2648

  // save for creating child table
2649 2650 2651
  superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1);
  if (NULL == superTbl->colsOfCreateChildTable) {
    errorPrint("%s() LN%d, Failed when calloc, size:%d",
2652
           __func__, __LINE__, len+1);
2653 2654 2655 2656
    taos_close(taos);
    exit(-1);
  }

2657
  snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
2658
  verbosePrint("%s() LN%d: %s\n",
2659
      __func__, __LINE__, superTbl->colsOfCreateChildTable);
2660

2661 2662
  if (superTbl->tagCount == 0) {
    errorPrint("%s() LN%d, super table tag count is %d\n",
2663
            __func__, __LINE__, superTbl->tagCount);
2664 2665
    return -1;
  }
2666

2667 2668 2669
  char tags[STRING_LEN] = "\0";
  int tagIndex;
  len = 0;
2670

2671 2672 2673 2674
  int lenOfTagOfOneRow = 0;
  len += snprintf(tags + len, STRING_LEN - len, "(");
  for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) {
    char* dataType = superTbl->tags[tagIndex].dataType;
2675

2676 2677
    if (strcasecmp(dataType, "BINARY") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
2678
              "BINARY", superTbl->tags[tagIndex].dataLen);
2679 2680 2681
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
2682
              "NCHAR", superTbl->tags[tagIndex].dataLen);
2683 2684 2685
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2686
              "INT");
2687 2688 2689
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2690
              "BIGINT");
2691 2692 2693
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2694
              "SMALLINT");
2695 2696 2697
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2698
              "TINYINT");
2699 2700 2701
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2702
              "BOOL");
2703 2704 2705
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2706
              "FLOAT");
2707 2708 2709
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2710
              "DOUBLE");
2711 2712 2713
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42;
    } else {
      taos_close(taos);
2714
      errorPrint("%s() LN%d, config error tag type : %s\n",
2715
         __func__, __LINE__, dataType);
2716
      exit(-1);
2717 2718
    }
  }
2719 2720 2721 2722 2723 2724 2725

  len -= 2;
  len += snprintf(tags + len, STRING_LEN - len, ")");

  superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow;

  snprintf(command, BUFFER_SIZE,
2726 2727
          "create table if not exists %s.%s (ts timestamp%s) tags %s",
          dbName, superTbl->sTblName, cols, tags);
2728 2729 2730
  verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command);

  if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2731 2732 2733
      errorPrint( "create supertable %s failed!\n\n",
              superTbl->sTblName);
      return -1;
2734 2735
  }
  debugPrint("create supertable %s success!\n\n", superTbl->sTblName);
2736 2737 2738
  return 0;
}

2739
static int createDatabasesAndStables() {
2740 2741 2742 2743
  TAOS * taos = NULL;
  int    ret = 0;
  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
  if (taos == NULL) {
2744
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2745
    return -1;
2746 2747 2748
  }
  char command[BUFFER_SIZE] = "\0";

2749
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2750 2751
    if (g_Dbs.db[i].drop) {
      sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
2752
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);
2753
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2754 2755 2756 2757
        taos_close(taos);
        return -1;
      }

2758
      int dataLen = 0;
2759
      dataLen += snprintf(command + dataLen,
2760
          BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2761

2762 2763
      if (g_Dbs.db[i].dbCfg.blocks > 0) {
        dataLen += snprintf(command + dataLen,
2764
            BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
2765 2766 2767
      }
      if (g_Dbs.db[i].dbCfg.cache > 0) {
        dataLen += snprintf(command + dataLen,
2768
            BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
2769 2770 2771
      }
      if (g_Dbs.db[i].dbCfg.days > 0) {
        dataLen += snprintf(command + dataLen,
2772
            BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
2773 2774 2775
      }
      if (g_Dbs.db[i].dbCfg.keep > 0) {
        dataLen += snprintf(command + dataLen,
2776
            BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
2777 2778 2779
      }
      if (g_Dbs.db[i].dbCfg.quorum > 1) {
        dataLen += snprintf(command + dataLen,
2780
            BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
2781 2782 2783
      }
      if (g_Dbs.db[i].dbCfg.replica > 0) {
        dataLen += snprintf(command + dataLen,
2784
            BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
2785 2786 2787
      }
      if (g_Dbs.db[i].dbCfg.update > 0) {
        dataLen += snprintf(command + dataLen,
2788
            BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
2789 2790 2791 2792 2793 2794 2795
      }
      //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
      //  dataLen += snprintf(command + dataLen,
      //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
      //}
      if (g_Dbs.db[i].dbCfg.minRows > 0) {
        dataLen += snprintf(command + dataLen,
2796
            BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
2797 2798 2799
      }
      if (g_Dbs.db[i].dbCfg.maxRows > 0) {
        dataLen += snprintf(command + dataLen,
2800
            BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
2801 2802 2803
      }
      if (g_Dbs.db[i].dbCfg.comp > 0) {
        dataLen += snprintf(command + dataLen,
2804
            BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
2805 2806 2807
      }
      if (g_Dbs.db[i].dbCfg.walLevel > 0) {
        dataLen += snprintf(command + dataLen,
2808
            BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
2809 2810 2811
      }
      if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
        dataLen += snprintf(command + dataLen,
2812
            BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
2813 2814 2815
      }
      if (g_Dbs.db[i].dbCfg.fsync > 0) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2816
                " fsync %d", g_Dbs.db[i].dbCfg.fsync);
2817 2818
      }
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
2819 2820
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                      "us", strlen("us")))) {
2821
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2822
                " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
2823 2824 2825 2826 2827 2828 2829 2830 2831
      }

      debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
        taos_close(taos);
        errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
        return -1;
      }
      printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
2832 2833
    }

2834
    debugPrint("%s() LN%d supertbl count:%"PRIu64"\n",
2835
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
2836 2837 2838

    int validStbCount = 0;

S
Shuduo Sang 已提交
2839
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
2840 2841 2842 2843 2844 2845 2846
      sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
              g_Dbs.db[i].superTbls[j].sTblName);
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);

      ret = queryDbExec(taos, command, NO_INSERT_TYPE, true);

      if ((ret != 0) || (g_Dbs.db[i].drop)) {
2847
        ret = createSuperTable(taos, g_Dbs.db[i].dbName,
2848
                &g_Dbs.db[i].superTbls[j]);
2849

2850
        if (0 != ret) {
S
Shuduo Sang 已提交
2851
          errorPrint("create super table %"PRIu64" failed!\n\n", j);
2852
          continue;
2853 2854 2855
        }
      }

2856
      ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
2857
                &g_Dbs.db[i].superTbls[j]);
2858 2859
      if (0 != ret) {
        errorPrint("\nget super table %s.%s info failed!\n\n",
2860
                g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName);
2861
        continue;
2862
      }
2863 2864

      validStbCount ++;
2865
    }
2866 2867

    g_Dbs.db[i].superTblCount = validStbCount;
2868 2869 2870 2871 2872 2873
  }

  taos_close(taos);
  return 0;
}

2874 2875
static void* createTable(void *sarg)
{
2876 2877
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
2878

S
Shuduo Sang 已提交
2879
  uint64_t  lastPrintTime = taosGetTimestampMs();
2880

2881
  int buff_len;
2882
  buff_len = BUFFER_SIZE / 8;
2883

2884
  char *buffer = calloc(buff_len, 1);
2885
  if (buffer == NULL) {
2886
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
2887 2888
    exit(-1);
  }
2889 2890 2891

  int len = 0;
  int batchNum = 0;
2892

2893
  verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n",
2894 2895
          __func__, __LINE__,
          pThreadInfo->start_table_from, pThreadInfo->end_table_to);
2896

2897
  for (uint64_t i = pThreadInfo->start_table_from;
2898
          i <= pThreadInfo->end_table_to; i++) {
2899
    if (0 == g_Dbs.use_metric) {
2900
      snprintf(buffer, buff_len,
2901
              "create table if not exists %s.%s%"PRIu64" %s;",
2902 2903 2904
              pThreadInfo->db_name,
              g_args.tb_prefix, i,
              pThreadInfo->cols);
2905
    } else {
2906 2907
      if (superTblInfo == NULL) {
        errorPrint("%s() LN%d, use metric, but super table info is NULL\n",
2908
                  __func__, __LINE__);
2909
        free(buffer);
2910 2911 2912 2913 2914 2915
        exit(-1);
      } else {
        if (0 == len) {
          batchNum = 0;
          memset(buffer, 0, buff_len);
          len += snprintf(buffer + len,
2916
                  buff_len - len, "create table ");
2917 2918 2919
        }
        char* tagsValBuf = NULL;
        if (0 == superTblInfo->tagSource) {
2920
          tagsValBuf = generateTagVaulesForStb(superTblInfo, i);
2921 2922
        } else {
          tagsValBuf = getTagValueFromTagSample(
2923 2924
                  superTblInfo,
                  i % superTblInfo->tagSampleCount);
2925 2926 2927 2928 2929 2930
        }
        if (NULL == tagsValBuf) {
          free(buffer);
          return NULL;
        }
        len += snprintf(buffer + len,
2931
                buff_len - len,
2932
                "if not exists %s.%s%"PRIu64" using %s.%s tags %s ",
2933 2934 2935
                pThreadInfo->db_name, superTblInfo->childTblPrefix,
                i, pThreadInfo->db_name,
                superTblInfo->sTblName, tagsValBuf);
2936 2937 2938
        free(tagsValBuf);
        batchNum++;
        if ((batchNum < superTblInfo->batchCreateTableNum)
2939 2940
                && ((buff_len - len)
                    >= (superTblInfo->lenOfTagOfOneRow + 256))) {
2941 2942
          continue;
        }
2943 2944 2945 2946
      }
    }

    len = 0;
2947
    verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
2948
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)){
2949
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2950
      free(buffer);
2951 2952 2953
      return NULL;
    }

S
Shuduo Sang 已提交
2954
    uint64_t  currentPrintTime = taosGetTimestampMs();
2955
    if (currentPrintTime - lastPrintTime > 30*1000) {
2956
      printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n",
2957
              pThreadInfo->threadID, pThreadInfo->start_table_from, i);
2958 2959 2960
      lastPrintTime = currentPrintTime;
    }
  }
2961

2962
  if (0 != len) {
2963
    verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer);
2964
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)) {
2965
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2966
    }
2967
  }
2968

2969 2970 2971 2972
  free(buffer);
  return NULL;
}

2973
static int startMultiThreadCreateChildTable(
S
Shuduo Sang 已提交
2974
        char* cols, int threads, uint64_t startFrom, int64_t ntables,
2975
        char* db_name, SSuperTable* superTblInfo) {
2976

2977
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
2978
  threadInfo *infos = calloc(1, threads * sizeof(threadInfo));
2979 2980 2981 2982 2983 2984 2985 2986 2987 2988

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

S
Shuduo Sang 已提交
2989
  int64_t a = ntables / threads;
2990 2991 2992 2993 2994
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

S
Shuduo Sang 已提交
2995
  int64_t b = 0;
2996
  b = ntables % threads;
2997

S
Shuduo Sang 已提交
2998
  for (int i = 0; i < threads; i++) {
2999 3000
    threadInfo *t_info = infos + i;
    t_info->threadID = i;
H
Hui Li 已提交
3001
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
3002
    t_info->superTblInfo = superTblInfo;
3003
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
3004
    t_info->taos = taos_connect(
3005 3006 3007 3008 3009
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
3010
    if (t_info->taos == NULL) {
3011
      errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n",
3012
         __func__, __LINE__, taos_errstr(NULL));
3013
      free(pids);
3014
      free(infos);
3015 3016
      return -1;
    }
3017 3018 3019 3020 3021

    t_info->start_table_from = startFrom;
    t_info->ntables = i<b?a+1:a;
    t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
    startFrom = t_info->end_table_to + 1;
3022
    t_info->use_metric = true;
3023
    t_info->cols = cols;
3024
    t_info->minDelay = UINT64_MAX;
3025 3026
    pthread_create(pids + i, NULL, createTable, t_info);
  }
3027

3028 3029 3030 3031 3032 3033 3034 3035 3036 3037
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
    threadInfo *t_info = infos + i;
    taos_close(t_info->taos);
  }

  free(pids);
3038
  free(infos);
3039 3040

  return 0;
3041 3042
}

3043
static void createChildTables() {
3044 3045
    char tblColsBuf[MAX_SQL_SIZE];
    int len;
3046

3047
  for (int i = 0; i < g_Dbs.dbCount; i++) {
3048 3049
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
3050
          // with super table
S
Shuduo Sang 已提交
3051
        for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
3052
          if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
3053
                || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
3054 3055
            continue;
          }
3056

3057
          verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
3058
                  g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
S
Shuduo Sang 已提交
3059
          uint64_t startFrom = 0;
3060 3061
          g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;

3062
          verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n",
3063
                  __func__, __LINE__, g_totalChildTables, startFrom);
3064
          startMultiThreadCreateChildTable(
3065 3066 3067 3068 3069
                g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
                g_Dbs.threadCountByCreateTbl,
                startFrom,
                g_Dbs.db[i].superTbls[j].childTblCount,
                g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
3070
        }
3071 3072
      }
    } else {
3073 3074
      // normal table
      len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
H
Haojun Liao 已提交
3075
      for (int j = 0; j < g_args.num_of_CPR; j++) {
3076 3077 3078 3079 3080 3081 3082 3083 3084 3085
          if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
                  || (strncasecmp(g_args.datatype[j],
                      "NCHAR", strlen("NCHAR")) == 0)) {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                      ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary);
          } else {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                      ", COL%d %s", j, g_args.datatype[j]);
          }
          len = strlen(tblColsBuf);
3086
      }
3087

3088
      snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");
3089

3090
      verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n",
3091 3092
              __func__, __LINE__,
              g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
3093
      startMultiThreadCreateChildTable(
3094 3095 3096 3097 3098 3099
            tblColsBuf,
            g_Dbs.threadCountByCreateTbl,
            0,
            g_args.num_of_tables,
            g_Dbs.db[i].dbName,
            NULL);
3100
    }
3101 3102 3103 3104 3105 3106
  }
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3107
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
3108 3109 3110
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
3111

3112 3113
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
3114
    printf("Failed to open tags file: %s, reason:%s\n",
3115
            superTblInfo->tagsFile, strerror(errno));
3116 3117 3118 3119 3120 3121 3122
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
3123

3124 3125 3126 3127 3128 3129 3130 3131 3132
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

3133
  while((readLen = tgetline(&line, &n, fp)) != -1) {
3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
3146
      char *tmp = realloc(tagDataBuf,
3147
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
3148 3149 3150
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
3151
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
3152
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

3172
#if 0
3173 3174 3175 3176
int readSampleFromJsonFileToMem(SSuperTable  * superTblInfo) {
  // TODO
  return 0;
}
3177
#endif
3178 3179 3180 3181

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3182
static int readSampleFromCsvFileToMem(
3183
        SSuperTable* superTblInfo) {
3184 3185 3186 3187
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
3188

3189 3190
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
3191 3192 3193
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
              superTblInfo->sampleFile, strerror(errno));
      return -1;
3194
  }
3195

3196
  assert(superTblInfo->sampleDataBuf);
3197
  memset(superTblInfo->sampleDataBuf, 0,
3198
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
3199
  while(1) {
3200
    readLen = tgetline(&line, &n, fp);
3201 3202
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
3203
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
3204
                superTblInfo->sampleFile, strerror(errno));
3205
        fclose(fp);
3206 3207 3208 3209
        return -1;
      }
      continue;
    }
3210

3211 3212 3213 3214 3215 3216 3217 3218 3219
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
3220
      printf("sample row len[%d] overflow define schema len[%"PRIu64"], so discard this row\n",
3221
              (int32_t)readLen, superTblInfo->lenOfOneRow);
3222 3223 3224
      continue;
    }

3225
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
3226
          line, readLen);
3227 3228 3229 3230 3231 3232 3233
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

3234
  fclose(fp);
3235 3236 3237 3238
  tmfree(line);
  return 0;
}

3239
static bool getColumnAndTagTypeFromInsertJsonFile(
3240
        cJSON* stbInfo, SSuperTable* superTbls) {
3241
  bool  ret = false;
3242

3243
  // columns
3244 3245
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
3246
    printf("ERROR: failed to read json, columns not found\n");
3247 3248 3249 3250 3251 3252
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
3253

3254
  int columnSize = cJSON_GetArraySize(columns);
3255
  if ((columnSize + 1/* ts */) > MAX_COLUMN_COUNT) {
3256
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
3257
            __func__, __LINE__, MAX_COLUMN_COUNT);
3258 3259 3260 3261 3262 3263
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
3264

3265
  //superTbls->columnCount = columnSize;
3266 3267 3268 3269 3270 3271 3272
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
3273
      count = countObj->valueint;
3274
    } else if (countObj && countObj->type != cJSON_Number) {
3275
      errorPrint("%s() LN%d, failed to read json, column count not found\n",
3276
          __func__, __LINE__);
3277 3278 3279 3280 3281
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3282
    // column info
3283 3284
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
3285 3286 3287
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d: failed to read json, column type not found\n",
3288
          __func__, __LINE__);
3289 3290
      goto PARSE_OVER;
    }
3291 3292
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3293

3294 3295
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3296
      columnCase.dataLen = dataLen->valueint;
3297
    } else if (dataLen && dataLen->type != cJSON_Number) {
3298
      debugPrint("%s() LN%d: failed to read json, column len not found\n",
3299
          __func__, __LINE__);
3300 3301 3302 3303
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
3304

3305
    for (int n = 0; n < count; ++n) {
3306
      tstrncpy(superTbls->columns[index].dataType,
3307
              columnCase.dataType, MAX_TB_NAME_SIZE);
3308
      superTbls->columns[index].dataLen = columnCase.dataLen;
3309 3310
      index++;
    }
3311
  }
3312

3313 3314
  if ((index + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n",
3315
            __func__, __LINE__, MAX_COLUMN_COUNT);
3316 3317 3318
    goto PARSE_OVER;
  }

3319
  superTbls->columnCount = index;
3320

3321 3322
  count = 1;
  index = 0;
3323
  // tags
3324 3325
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
3326
    errorPrint("%s() LN%d, failed to read json, tags not found\n",
3327
        __func__, __LINE__);
3328 3329
    goto PARSE_OVER;
  }
3330

3331 3332
  int tagSize = cJSON_GetArraySize(tags);
  if (tagSize > MAX_TAG_COUNT) {
3333
    errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n",
3334
        __func__, __LINE__, MAX_TAG_COUNT);
3335 3336
    goto PARSE_OVER;
  }
3337 3338

  //superTbls->tagCount = tagSize;
3339 3340 3341
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
3342

3343 3344 3345
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
3346
      count = countObj->valueint;
3347
    } else if (countObj && countObj->type != cJSON_Number) {
3348
      printf("ERROR: failed to read json, column count not found\n");
3349 3350 3351 3352 3353
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3354
    // column info
3355 3356
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
3357 3358 3359
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d, failed to read json, tag type not found\n",
3360
          __func__, __LINE__);
3361 3362
      goto PARSE_OVER;
    }
3363
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3364

3365 3366
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3367
      columnCase.dataLen = dataLen->valueint;
3368
    } else if (dataLen && dataLen->type != cJSON_Number) {
3369
      errorPrint("%s() LN%d, failed to read json, column len not found\n",
3370
          __func__, __LINE__);
3371 3372 3373
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
3374 3375
    }

3376
    for (int n = 0; n < count; ++n) {
3377
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType,
3378
          MAX_TB_NAME_SIZE);
3379
      superTbls->tags[index].dataLen = columnCase.dataLen;
3380 3381
      index++;
    }
3382
  }
3383 3384

  if (index > MAX_TAG_COUNT) {
3385
    errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n",
3386
        __func__, __LINE__, MAX_TAG_COUNT);
3387 3388 3389
    goto PARSE_OVER;
  }

3390 3391
  superTbls->tagCount = index;

3392 3393
  if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n",
3394
        __func__, __LINE__, MAX_COLUMN_COUNT);
3395 3396
    goto PARSE_OVER;
  }
3397 3398
  ret = true;

3399
PARSE_OVER:
3400 3401 3402 3403 3404 3405 3406 3407
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3408
    tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3409 3410 3411 3412
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3413
    tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE);
3414
  } else if (!host) {
3415
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3416
  } else {
3417
    printf("ERROR: failed to read json, host not found\n");
3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_Dbs.port = port->valueint;
  } else if (!port) {
    g_Dbs.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3430
    tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE);
3431
  } else if (!user) {
3432
    tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE);
3433 3434 3435 3436
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3437
    tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE);
3438
  } else if (!password) {
3439
    tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE);
3440 3441 3442 3443
  }

  cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
  if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
3444
    tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
3445
  } else if (!resultfile) {
3446
    tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
3447 3448 3449 3450 3451 3452 3453 3454
  }

  cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
  if (threads && threads->type == cJSON_Number) {
    g_Dbs.threadCount = threads->valueint;
  } else if (!threads) {
    g_Dbs.threadCount = 1;
  } else {
3455
    printf("ERROR: failed to read json, threads not found\n");
3456
    goto PARSE_OVER;
3457 3458
  }

3459 3460 3461 3462
  cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
  if (threads2 && threads2->type == cJSON_Number) {
    g_Dbs.threadCountByCreateTbl = threads2->valueint;
  } else if (!threads2) {
sangshuduo's avatar
sangshuduo 已提交
3463
    g_Dbs.threadCountByCreateTbl = 1;
3464
  } else {
3465
    errorPrint("%s() LN%d, failed to read json, threads2 not found\n",
3466
            __func__, __LINE__);
3467
    goto PARSE_OVER;
3468
  }
3469

3470 3471
  cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
  if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
3472 3473 3474 3475 3476
    if (gInsertInterval->valueint <0) {
      errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n",
            __func__, __LINE__);
      goto PARSE_OVER;
    }
3477 3478 3479 3480
    g_args.insert_interval = gInsertInterval->valueint;
  } else if (!gInsertInterval) {
    g_args.insert_interval = 0;
  } else {
3481
    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
3482
        __func__, __LINE__);
3483 3484
    goto PARSE_OVER;
  }
3485

3486 3487
  cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows");
  if (interlaceRows && interlaceRows->type == cJSON_Number) {
3488 3489 3490 3491 3492 3493
    if (interlaceRows->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;

    }
3494 3495
    g_args.interlace_rows = interlaceRows->valueint;
  } else if (!interlaceRows) {
3496
    g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3497
  } else {
3498
    errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
3499
        __func__, __LINE__);
3500
    goto PARSE_OVER;
3501
  }
3502

3503 3504
  cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
  if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
3505 3506 3507 3508 3509
    if (maxSqlLen->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
    }
3510 3511
    g_args.max_sql_len = maxSqlLen->valueint;
  } else if (!maxSqlLen) {
sangshuduo's avatar
sangshuduo 已提交
3512
    g_args.max_sql_len = (1024*1024);
3513
  } else {
3514
    errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
3515
        __func__, __LINE__);
3516 3517 3518 3519 3520
    goto PARSE_OVER;
  }

  cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
  if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
3521
    if (numRecPerReq->valueint <= 0) {
3522 3523 3524
      errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
3525
    } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) {
3526 3527 3528 3529 3530
      printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n",
              numRecPerReq->valueint, MAX_RECORDS_PER_REQ);
      printf("        number of records per request value will be set to %d\n\n",
              MAX_RECORDS_PER_REQ);
      prompt();
3531
      numRecPerReq->valueint = MAX_RECORDS_PER_REQ;
3532
    }
3533 3534
    g_args.num_of_RPR = numRecPerReq->valueint;
  } else if (!numRecPerReq) {
3535
    g_args.num_of_RPR = MAX_RECORDS_PER_REQ;
3536
  } else {
3537
    errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n",
3538
        __func__, __LINE__);
3539 3540 3541
    goto PARSE_OVER;
  }

3542
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3543
  if (answerPrompt
3544 3545
          && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3546 3547 3548 3549 3550 3551 3552 3553
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
    } else {
      g_args.answer_yes = false;
    }
  } else if (!answerPrompt) {
3554
    g_args.answer_yes = true;   // default is no, mean answer_yes.
3555
  } else {
3556
    errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n");
3557
    goto PARSE_OVER;
3558
  }
3559

3560 3561 3562 3563 3564 3565 3566 3567 3568 3569
  // rows per table need be less than insert batch
  if (g_args.interlace_rows > g_args.num_of_RPR) {
      printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
              g_args.interlace_rows, g_args.num_of_RPR);
      printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
              g_args.num_of_RPR);
      prompt();
      g_args.interlace_rows = g_args.num_of_RPR;
  }

3570 3571
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (!dbs || dbs->type != cJSON_Array) {
3572
    printf("ERROR: failed to read json, databases not found\n");
3573 3574 3575 3576 3577
    goto PARSE_OVER;
  }

  int dbSize = cJSON_GetArraySize(dbs);
  if (dbSize > MAX_DB_COUNT) {
3578
    errorPrint(
3579 3580
            "ERROR: failed to read json, databases size overflow, max database is %d\n",
            MAX_DB_COUNT);
3581 3582 3583 3584 3585 3586 3587 3588
    goto PARSE_OVER;
  }

  g_Dbs.dbCount = dbSize;
  for (int i = 0; i < dbSize; ++i) {
    cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
    if (dbinfos == NULL) continue;

3589
    // dbinfo
3590 3591
    cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
    if (!dbinfo || dbinfo->type != cJSON_Object) {
3592
      printf("ERROR: failed to read json, dbinfo not found\n");
3593 3594
      goto PARSE_OVER;
    }
3595

3596 3597
    cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
    if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
3598
      printf("ERROR: failed to read json, db name not found\n");
3599 3600
      goto PARSE_OVER;
    }
3601
    tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE);
3602 3603 3604

    cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
    if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
3605 3606
      if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) {
        g_Dbs.db[i].drop = true;
3607
      } else {
3608
        g_Dbs.db[i].drop = false;
3609
      }
3610
    } else if (!drop) {
3611
      g_Dbs.db[i].drop = g_args.drop_database;
3612
    } else {
3613
      errorPrint("%s() LN%d, failed to read json, drop input mistake\n",
3614
              __func__, __LINE__);
3615 3616 3617 3618
      goto PARSE_OVER;
    }

    cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
3619
    if (precision && precision->type == cJSON_String
3620
            && precision->valuestring != NULL) {
3621
      tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
3622
              MAX_DB_NAME_SIZE);
3623
    } else if (!precision) {
3624
      //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
3625 3626
      memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE);
    } else {
3627
      printf("ERROR: failed to read json, precision not found\n");
3628 3629 3630 3631 3632 3633 3634 3635 3636
      goto PARSE_OVER;
    }

    cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
    if (update && update->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.update = update->valueint;
    } else if (!update) {
      g_Dbs.db[i].dbCfg.update = -1;
    } else {
3637
      printf("ERROR: failed to read json, update not found\n");
3638 3639 3640 3641 3642 3643 3644 3645 3646
      goto PARSE_OVER;
    }

    cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
    if (replica && replica->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.replica = replica->valueint;
    } else if (!replica) {
      g_Dbs.db[i].dbCfg.replica = -1;
    } else {
3647
      printf("ERROR: failed to read json, replica not found\n");
3648 3649 3650 3651 3652 3653 3654 3655 3656
      goto PARSE_OVER;
    }

    cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
    if (keep && keep->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.keep = keep->valueint;
    } else if (!keep) {
      g_Dbs.db[i].dbCfg.keep = -1;
    } else {
3657 3658
     printf("ERROR: failed to read json, keep not found\n");
     goto PARSE_OVER;
3659
    }
3660

3661 3662 3663 3664 3665 3666
    cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
    if (days && days->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.days = days->valueint;
    } else if (!days) {
      g_Dbs.db[i].dbCfg.days = -1;
    } else {
3667 3668
     printf("ERROR: failed to read json, days not found\n");
     goto PARSE_OVER;
3669
    }
3670

3671 3672 3673 3674 3675 3676
    cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
    if (cache && cache->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cache = cache->valueint;
    } else if (!cache) {
      g_Dbs.db[i].dbCfg.cache = -1;
    } else {
3677 3678
     printf("ERROR: failed to read json, cache not found\n");
     goto PARSE_OVER;
3679
    }
3680

3681 3682 3683 3684 3685 3686
    cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
    if (blocks && blocks->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
    } else if (!blocks) {
      g_Dbs.db[i].dbCfg.blocks = -1;
    } else {
3687 3688
     printf("ERROR: failed to read json, block not found\n");
     goto PARSE_OVER;
3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704
    }

    //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
    //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
    //} else if (!maxtablesPerVnode) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
    //} else {
    // printf("failed to read json, maxtablesPerVnode not found");
    // goto PARSE_OVER;
    //}

    cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
    if (minRows && minRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
    } else if (!minRows) {
3705
      g_Dbs.db[i].dbCfg.minRows = 0;    // 0 means default
3706
    } else {
3707 3708
     printf("ERROR: failed to read json, minRows not found\n");
     goto PARSE_OVER;
3709 3710 3711 3712 3713 3714
    }

    cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
    if (maxRows && maxRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
    } else if (!maxRows) {
3715
      g_Dbs.db[i].dbCfg.maxRows = 0;    // 0 means default
3716
    } else {
3717 3718
     printf("ERROR: failed to read json, maxRows not found\n");
     goto PARSE_OVER;
3719 3720 3721 3722 3723 3724 3725 3726
    }

    cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
    if (comp && comp->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.comp = comp->valueint;
    } else if (!comp) {
      g_Dbs.db[i].dbCfg.comp = -1;
    } else {
3727 3728
     printf("ERROR: failed to read json, comp not found\n");
     goto PARSE_OVER;
3729 3730 3731 3732 3733 3734 3735 3736
    }

    cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
    if (walLevel && walLevel->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
    } else if (!walLevel) {
      g_Dbs.db[i].dbCfg.walLevel = -1;
    } else {
3737 3738
     printf("ERROR: failed to read json, walLevel not found\n");
     goto PARSE_OVER;
3739 3740
    }

3741 3742 3743 3744 3745 3746
    cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
    if (cacheLast && cacheLast->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
    } else if (!cacheLast) {
      g_Dbs.db[i].dbCfg.cacheLast = -1;
    } else {
3747 3748
     printf("ERROR: failed to read json, cacheLast not found\n");
     goto PARSE_OVER;
3749 3750
    }

3751 3752 3753 3754
    cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
    if (quorum && quorum->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
    } else if (!quorum) {
3755
      g_Dbs.db[i].dbCfg.quorum = 1;
3756
    } else {
3757 3758
     printf("failed to read json, quorum input mistake");
     goto PARSE_OVER;
3759 3760 3761 3762 3763 3764 3765 3766
    }

    cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
    if (fsync && fsync->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
    } else if (!fsync) {
      g_Dbs.db[i].dbCfg.fsync = -1;
    } else {
3767
      errorPrint("%s() LN%d, failed to read json, fsync input mistake\n",
3768
              __func__, __LINE__);
3769
      goto PARSE_OVER;
3770
    }
3771

3772
    // super_talbes
3773 3774
    cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
    if (!stables || stables->type != cJSON_Array) {
3775
      errorPrint("%s() LN%d, failed to read json, super_tables not found\n",
3776
              __func__, __LINE__);
3777
      goto PARSE_OVER;
3778 3779
    }

3780 3781
    int stbSize = cJSON_GetArraySize(stables);
    if (stbSize > MAX_SUPER_TABLE_COUNT) {
3782
      errorPrint(
3783 3784
              "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n",
              __func__, __LINE__, MAX_SUPER_TABLE_COUNT);
3785 3786 3787 3788 3789 3790 3791
      goto PARSE_OVER;
    }

    g_Dbs.db[i].superTblCount = stbSize;
    for (int j = 0; j < stbSize; ++j) {
      cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
      if (stbInfo == NULL) continue;
3792 3793

      // dbinfo
3794 3795
      cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
      if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) {
3796
        errorPrint("%s() LN%d, failed to read json, stb name not found\n",
3797
                __func__, __LINE__);
3798 3799
        goto PARSE_OVER;
      }
3800
      tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE);
3801

3802 3803
      cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
      if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
3804
        printf("ERROR: failed to read json, childtable_prefix not found\n");
3805 3806
        goto PARSE_OVER;
      }
3807
      tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE);
3808 3809

      cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null
3810
      if (autoCreateTbl
3811 3812
              && autoCreateTbl->type == cJSON_String
              && autoCreateTbl->valuestring != NULL) {
3813 3814 3815 3816 3817 3818 3819 3820 3821 3822
        if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
        } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        } else {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        }
      } else if (!autoCreateTbl) {
        g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      } else {
3823
        printf("ERROR: failed to read json, auto_create_table not found\n");
3824 3825
        goto PARSE_OVER;
      }
3826

3827 3828 3829 3830
      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
      } else if (!batchCreateTbl) {
3831
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
3832
      } else {
3833
        printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
3834
        goto PARSE_OVER;
3835
      }
3836 3837

      cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
3838
      if (childTblExists
3839 3840
              && childTblExists->type == cJSON_String
              && childTblExists->valuestring != NULL) {
3841 3842
        if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3))
            && (g_Dbs.db[i].drop == false)) {
3843
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
3844
        } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2)
3845
              || (g_Dbs.db[i].drop == true))) {
3846 3847 3848 3849 3850 3851 3852
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        } else {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        }
      } else if (!childTblExists) {
        g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
      } else {
3853
        errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n",
3854
                __func__, __LINE__);
3855 3856
        goto PARSE_OVER;
      }
3857

3858 3859
      cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
      if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
3860
        errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n",
3861
                __func__, __LINE__);
3862 3863 3864 3865 3866
        goto PARSE_OVER;
      }
      g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

      cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
3867
      if (dataSource && dataSource->type == cJSON_String
3868
              && dataSource->valuestring != NULL) {
3869
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
3870
                dataSource->valuestring, MAX_DB_NAME_SIZE);
3871
      } else if (!dataSource) {
3872
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE);
3873
      } else {
3874
        errorPrint("%s() LN%d, failed to read json, data_source not found\n",
3875
            __func__, __LINE__);
3876 3877 3878
        goto PARSE_OVER;
      }

3879
      cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt
3880
      if (insertMode && insertMode->type == cJSON_String
3881
              && insertMode->valuestring != NULL) {
3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892
        if (0 == strcasecmp(insertMode->valuestring, "taosc")) {
            g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE;
        } else if (0 == strcasecmp(insertMode->valuestring, "rest")) {
            g_Dbs.db[i].superTbls[j].insertMode = REST_IFACE;
        } else if (0 == strcasecmp(insertMode->valuestring, "stmt")) {
            g_Dbs.db[i].superTbls[j].insertMode = STMT_IFACE;
        } else {
            errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n",
                    __func__, __LINE__, insertMode->valuestring);
            goto PARSE_OVER;
        }
3893
      } else if (!insertMode) {
3894
        g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE;
3895
      } else {
3896
        errorPrint("%s", "failed to read json, insert_mode not found\n");
3897 3898 3899
        goto PARSE_OVER;
      }

3900
      cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
3901 3902
      if ((childTbl_limit) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3903
        if (childTbl_limit->type != cJSON_Number) {
3904 3905
            printf("ERROR: failed to read json, childtable_limit\n");
            goto PARSE_OVER;
3906 3907 3908
        }
        g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
      } else {
3909
        g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid.
3910 3911 3912
      }

      cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
3913 3914
      if ((childTbl_offset) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3915
        if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) {
3916 3917
            printf("ERROR: failed to read json, childtable_offset\n");
            goto PARSE_OVER;
3918 3919 3920 3921 3922 3923
        }
        g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblOffset = 0;
      }

3924 3925
      cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
      if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
3926
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
3927
                ts->valuestring, MAX_DB_NAME_SIZE);
3928
      } else if (!ts) {
3929
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
3930
                "now", MAX_DB_NAME_SIZE);
3931
      } else {
3932
        printf("ERROR: failed to read json, start_timestamp not found\n");
3933 3934
        goto PARSE_OVER;
      }
3935

3936 3937 3938 3939
      cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
      if (timestampStep && timestampStep->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
      } else if (!timestampStep) {
3940
        g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
3941
      } else {
3942
        printf("ERROR: failed to read json, timestamp_step not found\n");
3943 3944 3945 3946
        goto PARSE_OVER;
      }

      cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
3947
      if (sampleFormat && sampleFormat->type
3948
              == cJSON_String && sampleFormat->valuestring != NULL) {
3949
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
3950
                sampleFormat->valuestring, MAX_DB_NAME_SIZE);
3951
      } else if (!sampleFormat) {
3952
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE);
3953
      } else {
3954
        printf("ERROR: failed to read json, sample_format not found\n");
3955
        goto PARSE_OVER;
3956
      }
3957

3958
      cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
3959 3960
      if (sampleFile && sampleFile->type == cJSON_String
          && sampleFile->valuestring != NULL) {
3961
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
3962
                sampleFile->valuestring, MAX_FILE_NAME_LEN);
3963 3964 3965
      } else if (!sampleFile) {
        memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
      } else {
3966
        printf("ERROR: failed to read json, sample_file not found\n");
3967
        goto PARSE_OVER;
3968
      }
3969

3970 3971
      cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
      if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) {
3972
        tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
3973
                tagsFile->valuestring, MAX_FILE_NAME_LEN);
3974 3975 3976 3977 3978 3979 3980 3981 3982
        if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
          g_Dbs.db[i].superTbls[j].tagSource = 0;
        } else {
          g_Dbs.db[i].superTbls[j].tagSource = 1;
        }
      } else if (!tagsFile) {
        memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
        g_Dbs.db[i].superTbls[j].tagSource = 0;
      } else {
3983
        printf("ERROR: failed to read json, tags_file not found\n");
3984 3985
        goto PARSE_OVER;
      }
3986

sangshuduo's avatar
sangshuduo 已提交
3987 3988 3989
      cJSON* stbMaxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
      if (stbMaxSqlLen && stbMaxSqlLen->type == cJSON_Number) {
        int32_t len = stbMaxSqlLen->valueint;
3990 3991
        if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
          len = TSDB_MAX_ALLOWED_SQL_LEN;
3992 3993
        } else if (len < 5) {
          len = 5;
3994
        }
3995 3996
        g_Dbs.db[i].superTbls[j].maxSqlLen = len;
      } else if (!maxSqlLen) {
3997
        g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len;
3998
      } else {
sangshuduo's avatar
sangshuduo 已提交
3999
        errorPrint("%s() LN%d, failed to read json, stbMaxSqlLen input mistake\n",
4000
            __func__, __LINE__);
4001
        goto PARSE_OVER;
4002
      }
4003
/*
4004 4005 4006 4007 4008
      cJSON *multiThreadWriteOneTbl =
          cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
      if (multiThreadWriteOneTbl
              && multiThreadWriteOneTbl->type == cJSON_String
              && multiThreadWriteOneTbl->valuestring != NULL) {
4009 4010 4011 4012
        if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
        } else {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
4013
        }
4014 4015 4016
      } else if (!multiThreadWriteOneTbl) {
        g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
      } else {
4017
        printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
4018 4019
        goto PARSE_OVER;
      }
4020
*/
sangshuduo's avatar
sangshuduo 已提交
4021 4022 4023
      cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows");
      if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) {
        if (stbInterlaceRows->valueint < 0) {
4024 4025 4026 4027
          errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n",
            __func__, __LINE__);
          goto PARSE_OVER;
        }
sangshuduo's avatar
sangshuduo 已提交
4028
        g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint;
4029 4030
        // rows per table need be less than insert batch
        if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) {
4031
          printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
4032
                  i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR);
4033
          printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
4034
                  g_args.num_of_RPR);
4035
          prompt();
4036 4037
          g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR;
        }
sangshuduo's avatar
sangshuduo 已提交
4038
      } else if (!stbInterlaceRows) {
4039
        g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
4040
      } else {
4041
        errorPrint(
4042 4043
                "%s() LN%d, failed to read json, interlace rows input mistake\n",
                __func__, __LINE__);
4044
        goto PARSE_OVER;
4045
      }
4046 4047 4048

      cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
      if (disorderRatio && disorderRatio->type == cJSON_Number) {
4049 4050 4051 4052 4053 4054
        if (disorderRatio->valueint > 50)
          disorderRatio->valueint = 50;

        if (disorderRatio->valueint < 0)
          disorderRatio->valueint = 0;

4055 4056 4057 4058
        g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
      } else if (!disorderRatio) {
        g_Dbs.db[i].superTbls[j].disorderRatio = 0;
      } else {
4059
        printf("ERROR: failed to read json, disorderRatio not found\n");
4060
        goto PARSE_OVER;
4061
      }
4062 4063 4064 4065 4066 4067 4068

      cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
      if (disorderRange && disorderRange->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
      } else if (!disorderRange) {
        g_Dbs.db[i].superTbls[j].disorderRange = 1000;
      } else {
4069
        printf("ERROR: failed to read json, disorderRange not found\n");
4070 4071
        goto PARSE_OVER;
      }
4072

4073 4074
      cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
      if (insertRows && insertRows->type == cJSON_Number) {
4075 4076 4077 4078 4079
        if (insertRows->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4080 4081 4082 4083
        g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
      } else if (!insertRows) {
        g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
      } else {
4084
        errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
4085
                __func__, __LINE__);
4086 4087 4088 4089 4090 4091
        goto PARSE_OVER;
      }

      cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
      if (insertInterval && insertInterval->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
4092 4093 4094 4095 4096
        if (insertInterval->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4097
      } else if (!insertInterval) {
4098
        verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n",
4099
                __func__, __LINE__, g_args.insert_interval);
4100 4101
        g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
      } else {
4102
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
4103
                __func__, __LINE__);
4104 4105 4106
        goto PARSE_OVER;
      }

4107
      int retVal = getColumnAndTagTypeFromInsertJsonFile(
4108
              stbInfo, &g_Dbs.db[i].superTbls[j]);
4109 4110
      if (false == retVal) {
        goto PARSE_OVER;
4111 4112
      }
    }
4113 4114 4115 4116
  }

  ret = true;

4117
PARSE_OVER:
4118 4119 4120 4121 4122 4123 4124 4125
  return ret;
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
4126
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
4127 4128 4129 4130
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
4131
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE);
4132
  } else if (!host) {
4133
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
4134
  } else {
4135
    printf("ERROR: failed to read json, host not found\n");
4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
4148
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE);
4149
  } else if (!user) {
4150
    tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ;
4151 4152 4153 4154
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
4155
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE);
4156
  } else if (!password) {
4157
    tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);;
4158
  }
H
hzcheng 已提交
4159

4160
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
4161
  if (answerPrompt && answerPrompt->type == cJSON_String
4162
          && answerPrompt->valuestring != NULL) {
4163 4164 4165 4166
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
4167
    } else {
4168
      g_args.answer_yes = false;
H
hzcheng 已提交
4169
    }
4170 4171 4172
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
4173
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
4174
    goto PARSE_OVER;
4175
  }
4176

4177 4178
  cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
  if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
4179
    if (gQueryTimes->valueint <= 0) {
4180 4181
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4182 4183
      goto PARSE_OVER;
    }
4184 4185 4186 4187
    g_args.query_times = gQueryTimes->valueint;
  } else if (!gQueryTimes) {
    g_args.query_times = 1;
  } else {
4188
    errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4189
        __func__, __LINE__);
4190 4191 4192
    goto PARSE_OVER;
  }

4193 4194
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
4195
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE);
4196
  } else if (!dbs) {
4197
    printf("ERROR: failed to read json, databases not found\n");
4198
    goto PARSE_OVER;
H
hzcheng 已提交
4199
  }
4200 4201 4202

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
4203
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
4204
  } else if (!queryMode) {
4205
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
4206
  } else {
4207
    printf("ERROR: failed to read json, query_mode not found\n");
4208
    goto PARSE_OVER;
H
hzcheng 已提交
4209
  }
4210

4211
  // specified_table_query
4212 4213
  cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!specifiedQuery) {
4214
    g_queryInfo.specifiedQueryInfo.concurrent = 1;
4215 4216
    g_queryInfo.specifiedQueryInfo.sqlCount = 0;
  } else if (specifiedQuery->type != cJSON_Object) {
4217
    printf("ERROR: failed to read json, super_table_query not found\n");
4218
    goto PARSE_OVER;
4219
  } else {
4220 4221 4222 4223 4224
    cJSON* queryInterval = cJSON_GetObjectItem(specifiedQuery, "query_interval");
    if (queryInterval && queryInterval->type == cJSON_Number) {
      g_queryInfo.specifiedQueryInfo.queryInterval = queryInterval->valueint;
    } else if (!queryInterval) {
      g_queryInfo.specifiedQueryInfo.queryInterval = 0;
4225 4226
    }

4227
    cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery,
4228
        "query_times");
4229
    if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
4230
      if (specifiedQueryTimes->valueint <= 0) {
4231 4232
        errorPrint(
                "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4233
          __func__, __LINE__, specifiedQueryTimes->valueint);
4234 4235 4236
        goto PARSE_OVER;

      }
4237 4238 4239 4240
      g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint;
    } else if (!specifiedQueryTimes) {
      g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times;
    } else {
4241
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4242
          __func__, __LINE__);
4243
      goto PARSE_OVER;
4244
    }
4245

4246
    cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent");
4247
    if (concurrent && concurrent->type == cJSON_Number) {
4248
      if (concurrent->valueint <= 0) {
4249 4250
        errorPrint(
                "%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n",
4251 4252
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount,
4253
              g_queryInfo.specifiedQueryInfo.concurrent);
4254 4255
        goto PARSE_OVER;
      }
4256
      g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint;
4257
    } else if (!concurrent) {
4258
      g_queryInfo.specifiedQueryInfo.concurrent = 1;
4259
    }
4260

4261 4262 4263 4264 4265 4266 4267
    cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode");
    if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String
        && specifiedAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE;
4268
      } else {
4269
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4270
            __func__, __LINE__);
4271 4272 4273
        goto PARSE_OVER;
      }
    } else {
4274
      g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
4275
    }
4276

4277
    cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval");
4278
    if (interval && interval->type == cJSON_Number) {
4279
      g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint;
4280
    } else if (!interval) {
4281 4282
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4283
      g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000;
4284
    }
4285

4286
    cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart");
4287
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
4288
      if (0 == strcmp("yes", restart->valuestring)) {
4289
        g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4290
      } else if (0 == strcmp("no", restart->valuestring)) {
4291
        g_queryInfo.specifiedQueryInfo.subscribeRestart = false;
4292
      } else {
4293
        printf("ERROR: failed to read json, subscribe restart error\n");
4294 4295 4296
        goto PARSE_OVER;
      }
    } else {
4297
      g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4298
    }
4299

4300
    cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress");
4301
    if (keepProgress
4302 4303
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
4304
      if (0 == strcmp("yes", keepProgress->valuestring)) {
4305
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1;
4306
      } else if (0 == strcmp("no", keepProgress->valuestring)) {
4307
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4308
      } else {
4309
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4310 4311 4312
        goto PARSE_OVER;
      }
    } else {
4313
      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4314 4315
    }

4316
    // sqls
4317
    cJSON* superSqls = cJSON_GetObjectItem(specifiedQuery, "sqls");
4318
    if (!superSqls) {
4319
      g_queryInfo.specifiedQueryInfo.sqlCount = 0;
4320
    } else if (superSqls->type != cJSON_Array) {
4321
      errorPrint("%s() LN%d, failed to read json, super sqls not found\n",
4322
          __func__, __LINE__);
4323
      goto PARSE_OVER;
4324
    } else {
4325 4326
      int superSqlSize = cJSON_GetArraySize(superSqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4327
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
4328
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4329 4330
        goto PARSE_OVER;
      }
4331

4332
      g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize;
4333 4334 4335
      for (int j = 0; j < superSqlSize; ++j) {
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
        if (sql == NULL) continue;
4336

4337 4338
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
4339
          printf("ERROR: failed to read json, sql not found\n");
4340 4341
          goto PARSE_OVER;
        }
4342 4343
        tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j],
                sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
H
hzcheng 已提交
4344

4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356
        cJSON* resubAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume");
        if (resubAfterConsume
                && resubAfterConsume->type == cJSON_Number) {
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j]
                = resubAfterConsume->valueint;
        } else if (!resubAfterConsume) {
            //printf("failed to read json, subscribe interval no found\n");
            //goto PARSE_OVER;
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = 1;
        }

4357
        cJSON *result = cJSON_GetObjectItem(sql, "result");
4358 4359 4360 4361
        if ((NULL != result) && (result->type == cJSON_String)
                && (result->valuestring != NULL)) {
          tstrncpy(g_queryInfo.specifiedQueryInfo.result[j],
                  result->valuestring, MAX_FILE_NAME_LEN);
4362
        } else if (NULL == result) {
4363 4364
          memset(g_queryInfo.specifiedQueryInfo.result[j],
                  0, MAX_FILE_NAME_LEN);
4365
        } else {
4366
          printf("ERROR: failed to read json, super query result file not found\n");
4367
          goto PARSE_OVER;
4368
        }
4369 4370 4371
      }
    }
  }
4372

4373
  // super_table_query
4374 4375
  cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!superQuery) {
4376
    g_queryInfo.superQueryInfo.threadCnt = 1;
4377 4378
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
4379
    printf("ERROR: failed to read json, sub_table_query not found\n");
4380 4381
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
4382
  } else {
4383
    cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval");
4384
    if (subrate && subrate->type == cJSON_Number) {
4385
      g_queryInfo.superQueryInfo.queryInterval = subrate->valueint;
4386
    } else if (!subrate) {
4387
      g_queryInfo.superQueryInfo.queryInterval = 0;
4388 4389 4390 4391
    }

    cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
    if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
4392 4393
      if (superQueryTimes->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4394
          __func__, __LINE__, superQueryTimes->valueint);
4395 4396
        goto PARSE_OVER;
      }
4397 4398 4399 4400
      g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
    } else if (!superQueryTimes) {
      g_queryInfo.superQueryInfo.queryTimes = g_args.query_times;
    } else {
4401
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4402
          __func__, __LINE__);
4403
      goto PARSE_OVER;
4404
    }
4405

4406
    cJSON* threads = cJSON_GetObjectItem(superQuery, "threads");
4407
    if (threads && threads->type == cJSON_Number) {
4408 4409 4410 4411 4412 4413
      if (threads->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, threads input mistake\n",
          __func__, __LINE__);
        goto PARSE_OVER;

      }
4414
      g_queryInfo.superQueryInfo.threadCnt = threads->valueint;
4415
    } else if (!threads) {
4416
      g_queryInfo.superQueryInfo.threadCnt = 1;
4417
    }
4418

4419
    //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count");
4420
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
4421
    //  g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint;
4422
    //} else if (!subTblCnt) {
4423
    //  g_queryInfo.superQueryInfo.childTblCount = 0;
4424
    //}
4425

4426
    cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname");
4427 4428 4429
    if (stblname && stblname->type == cJSON_String
        && stblname->valuestring != NULL) {
      tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring,
4430
          MAX_TB_NAME_SIZE);
4431
    } else {
4432
      errorPrint("%s() LN%d, failed to read json, super table name input error\n",
4433
          __func__, __LINE__);
4434 4435
      goto PARSE_OVER;
    }
4436

4437 4438 4439 4440 4441 4442 4443
    cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode");
    if (superAsyncMode && superAsyncMode->type == cJSON_String
        && superAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE;
4444
      } else {
4445
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4446
            __func__, __LINE__);
4447 4448 4449
        goto PARSE_OVER;
      }
    } else {
4450
      g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
4451
    }
4452

4453 4454 4455 4456 4457 4458 4459 4460 4461
    cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval");
    if (superInterval && superInterval->type == cJSON_Number) {
      if (superInterval->valueint < 0) {
        errorPrint("%s() LN%d, failed to read json, interval input mistake\n",
            __func__, __LINE__);
        goto PARSE_OVER;
      }
      g_queryInfo.superQueryInfo.subscribeInterval = superInterval->valueint;
    } else if (!superInterval) {
4462 4463
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4464
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
4465
    }
4466

4467
    cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart");
4468 4469
    if (subrestart && subrestart->type == cJSON_String
        && subrestart->valuestring != NULL) {
4470
      if (0 == strcmp("yes", subrestart->valuestring)) {
4471
        g_queryInfo.superQueryInfo.subscribeRestart = true;
4472
      } else if (0 == strcmp("no", subrestart->valuestring)) {
4473
        g_queryInfo.superQueryInfo.subscribeRestart = false;
4474
      } else {
4475
        printf("ERROR: failed to read json, subscribe restart error\n");
4476 4477 4478
        goto PARSE_OVER;
      }
    } else {
4479
      g_queryInfo.superQueryInfo.subscribeRestart = true;
4480
    }
4481

4482 4483 4484 4485 4486
    cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
    if (superkeepProgress &&
            superkeepProgress->type == cJSON_String
            && superkeepProgress->valuestring != NULL) {
      if (0 == strcmp("yes", superkeepProgress->valuestring)) {
4487
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
4488
      } else if (0 == strcmp("no", superkeepProgress->valuestring)) {
4489
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4490
      } else {
4491
        printf("ERROR: failed to read json, subscribe super table keepProgress error\n");
4492 4493 4494
        goto PARSE_OVER;
      }
    } else {
4495
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4496
    }
4497

4498
    // sqls
4499
    cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls");
4500
    if (!subsqls) {
4501
      g_queryInfo.superQueryInfo.sqlCount = 0;
4502
    } else if (subsqls->type != cJSON_Array) {
4503
      errorPrint("%s() LN%d: failed to read json, super sqls not found\n",
4504
          __func__, __LINE__);
4505
      goto PARSE_OVER;
4506
    } else {
4507 4508
      int superSqlSize = cJSON_GetArraySize(subsqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4509
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
4510
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4511 4512
        goto PARSE_OVER;
      }
4513

4514
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
4515
      for (int j = 0; j < superSqlSize; ++j) {
4516 4517
        cJSON* sql = cJSON_GetArrayItem(subsqls, j);
        if (sql == NULL) continue;
4518

4519
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
4520 4521 4522
        if (!sqlStr || sqlStr->type != cJSON_String
            || sqlStr->valuestring == NULL) {
          errorPrint("%s() LN%d, failed to read json, sql not found\n",
4523
              __func__, __LINE__);
4524 4525
          goto PARSE_OVER;
        }
4526
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring,
4527
            MAX_QUERY_SQL_LENGTH);
4528

4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540
        cJSON* superResubAfterConsume =
            cJSON_GetObjectItem(sql, "resubAfterConsume");
        if (superResubAfterConsume
                && superResubAfterConsume->type == cJSON_Number) {
            g_queryInfo.superQueryInfo.resubAfterConsume[j] =
                superResubAfterConsume->valueint;
        } else if (!superResubAfterConsume) {
            //printf("failed to read json, subscribe interval no found\n");
            //goto PARSE_OVER;
            g_queryInfo.superQueryInfo.resubAfterConsume[j] = 1;
        }

4541
        cJSON *result = cJSON_GetObjectItem(sql, "result");
4542 4543 4544
        if (result != NULL && result->type == cJSON_String
            && result->valuestring != NULL){
          tstrncpy(g_queryInfo.superQueryInfo.result[j],
4545
              result->valuestring, MAX_FILE_NAME_LEN);
4546
        } else if (NULL == result) {
4547
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4548
        }  else {
4549
          errorPrint("%s() LN%d, failed to read json, sub query result file not found\n",
4550
              __func__, __LINE__);
4551
          goto PARSE_OVER;
4552
        }
4553 4554
      }
    }
H
hzcheng 已提交
4555 4556
  }

4557
  ret = true;
H
hzcheng 已提交
4558

4559
PARSE_OVER:
4560 4561
  return ret;
}
H
hzcheng 已提交
4562

4563
static bool getInfoFromJsonFile(char* file) {
4564
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
4565

4566 4567 4568 4569 4570
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
4571

4572
  bool  ret = false;
4573
  int   maxLen = 6400000;
4574 4575 4576 4577 4578 4579 4580
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
4581
  }
H
Hui Li 已提交
4582

4583 4584 4585
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
4586
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
4587 4588
    goto PARSE_OVER;
  }
H
Hui Li 已提交
4589

4590 4591 4592
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
4593
      g_args.test_mode = INSERT_TEST;
4594
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
4595
      g_args.test_mode = QUERY_TEST;
4596
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
4597
      g_args.test_mode = SUBSCRIBE_TEST;
4598
    } else {
4599
      printf("ERROR: failed to read json, filetype not support\n");
4600 4601 4602
      goto PARSE_OVER;
    }
  } else if (!filetype) {
4603
    g_args.test_mode = INSERT_TEST;
4604
  } else {
4605
    printf("ERROR: failed to read json, filetype not found\n");
4606 4607
    goto PARSE_OVER;
  }
H
hzcheng 已提交
4608

4609
  if (INSERT_TEST == g_args.test_mode) {
4610
    ret = getMetaFromInsertJsonFile(root);
4611
  } else if ((QUERY_TEST == g_args.test_mode)
4612
          || (SUBSCRIBE_TEST == g_args.test_mode)) {
4613 4614
    ret = getMetaFromQueryJsonFile(root);
  } else {
4615
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n",
4616
            __func__, __LINE__);
4617
    goto PARSE_OVER;
4618
  }
4619

4620
PARSE_OVER:
4621 4622
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
4623
  fclose(fp);
4624 4625
  return ret;
}
H
hzcheng 已提交
4626

4627
static void prepareSampleData() {
4628
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4629 4630 4631
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
        (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]);
S
Shuaiqiang Chang 已提交
4632
      }
4633 4634 4635 4636
    }
  }
}

4637
static void postFreeResource() {
4638
  tmfclose(g_fpOfInsertResult);
4639
  for (int i = 0; i < g_Dbs.dbCount; i++) {
S
Shuduo Sang 已提交
4640
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
4641 4642 4643
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
4644
      }
4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4657
    }
4658 4659
  }
}
S
Shuaiqiang Chang 已提交
4660

4661 4662 4663
static int getRowDataFromSample(
        char* dataBuf, int64_t maxLen, int64_t timestamp,
      SSuperTable* superTblInfo, int64_t* sampleUsePos) {
4664
  if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
4665
/*    int ret = readSampleFromCsvFileToMem(superTblInfo);
4666
    if (0 != ret) {
4667
      tmfree(superTblInfo->sampleDataBuf);
4668
      superTblInfo->sampleDataBuf = NULL;
4669 4670
      return -1;
    }
4671
*/
4672 4673
    *sampleUsePos = 0;
  }
S
Shuaiqiang Chang 已提交
4674

4675
  int    dataLen = 0;
4676

4677
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
4678
          "(%" PRId64 ", ", timestamp);
4679
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
4680
          "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
4681
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
S
Shuaiqiang Chang 已提交
4682

4683
  (*sampleUsePos)++;
4684

4685 4686
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4687

4688 4689 4690 4691
static int64_t generateStbRowData(
        SSuperTable* stbInfo,
        char* recBuf, int64_t timestamp
        ) {
4692
  int64_t   dataLen = 0;
4693
  char  *pstr = recBuf;
4694
  int64_t maxLen = MAX_DATA_SIZE;
4695

4696 4697
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
          "(%" PRId64 ",", timestamp);
4698

4699
  for (int i = 0; i < stbInfo->columnCount; i++) {
4700 4701 4702 4703
    if ((0 == strncasecmp(stbInfo->columns[i].dataType,
                    "BINARY", strlen("BINARY")))
            || (0 == strncasecmp(stbInfo->columns[i].dataType,
                    "NCHAR", strlen("NCHAR")))) {
4704
      if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
4705
        errorPrint( "binary or nchar length overflow, max size:%u\n",
4706
                (uint32_t)TSDB_MAX_BINARY_LEN);
4707
        return -1;
4708
      }
4709

4710 4711
      char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
      if (NULL == buf) {
4712
        errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
4713
        return -1;
4714 4715
      }
      rand_string(buf, stbInfo->columns[i].dataLen);
4716
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf);
4717
      tmfree(buf);
4718
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4719
                "INT", 3)) {
4720
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4721
              "%d,", rand_int());
4722
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4723
                "BIGINT", 6)) {
4724
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4725
              "%"PRId64",", rand_bigint());
4726
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4727
                "FLOAT", 5)) {
4728
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4729
              "%f,", rand_float());
4730
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4731
                "DOUBLE", 6)) {
4732
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4733
              "%f,", rand_double());
4734
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4735
                "SMALLINT", 8)) {
4736
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4737
          "%d,", rand_smallint());
4738
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4739
          "TINYINT", strlen("TINYINT"))) {
4740
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4741
          "%d,", rand_tinyint());
4742
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4743
          "BOOL", strlen("BOOL"))) {
4744
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4745
          "%d,", rand_bool());
4746
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4747
          "TIMESTAMP", strlen("TIMESTAMP"))) {
4748
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4749
          "%"PRId64",", rand_bigint());
4750
    }  else {
4751
      errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType);
4752
      return -1;
4753
    }
S
Shuaiqiang Chang 已提交
4754
  }
4755

4756
  dataLen -= 1;
4757
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")");
4758

4759 4760 4761
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

  return strlen(recBuf);
4762
}
S
Shuaiqiang Chang 已提交
4763

4764
static int64_t generateData(char *recBuf, char **data_type,
4765
        int64_t timestamp, int lenOfBinary) {
4766 4767
  memset(recBuf, 0, MAX_DATA_SIZE);
  char *pstr = recBuf;
4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781
  pstr += sprintf(pstr, "(%" PRId64, timestamp);
  int c = 0;

  for (; c < MAX_NUM_DATATYPE; c++) {
    if (data_type[c] == NULL) {
      break;
    }
  }

  if (0 == c) {
    perror("data type error!");
    exit(-1);
  }

H
Haojun Liao 已提交
4782
  for (int i = 0; i < c; i++) {
4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793
    if (strcasecmp(data_type[i % c], "TINYINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_tinyint() );
    } else if (strcasecmp(data_type[i % c], "SMALLINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_smallint());
    } else if (strcasecmp(data_type[i % c], "INT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_int());
    } else if (strcasecmp(data_type[i % c], "BIGINT") == 0) {
      pstr += sprintf(pstr, ",%" PRId64, rand_bigint());
    } else if (strcasecmp(data_type[i % c], "FLOAT") == 0) {
      pstr += sprintf(pstr, ",%10.4f", rand_float());
    } else if (strcasecmp(data_type[i % c], "DOUBLE") == 0) {
4794
      double t = rand_double();
4795 4796
      pstr += sprintf(pstr, ",%20.8f", t);
    } else if (strcasecmp(data_type[i % c], "BOOL") == 0) {
4797
      bool b = rand_bool() & 1;
4798 4799
      pstr += sprintf(pstr, ",%s", b ? "true" : "false");
    } else if (strcasecmp(data_type[i % c], "BINARY") == 0) {
4800 4801
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4802
      pstr += sprintf(pstr, ",\"%s\"", s);
4803
      free(s);
4804
    } else if (strcasecmp(data_type[i % c], "NCHAR") == 0) {
4805 4806
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4807
      pstr += sprintf(pstr, ",\"%s\"", s);
4808 4809 4810
      free(s);
    }

4811
    if (strlen(recBuf) > MAX_DATA_SIZE) {
4812 4813 4814 4815 4816 4817 4818
      perror("column length too long, abort");
      exit(-1);
    }
  }

  pstr += sprintf(pstr, ")");

4819 4820
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

4821
  return (int32_t)strlen(recBuf);
4822 4823
}

4824
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
4825 4826
  char* sampleDataBuf = NULL;

4827
  sampleDataBuf = calloc(
4828
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
4829
  if (sampleDataBuf == NULL) {
4830
      errorPrint("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n",
4831 4832 4833 4834
              __func__, __LINE__,
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE,
              strerror(errno));
      return -1;
4835
  }
4836

4837 4838 4839 4840
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
4841 4842 4843 4844 4845
      errorPrint("%s() LN%d, read sample from csv file failed.\n",
          __func__, __LINE__);
      tmfree(sampleDataBuf);
      superTblInfo->sampleDataBuf = NULL;
      return -1;
H
Hui Li 已提交
4846
  }
4847

4848 4849 4850
  return 0;
}

S
Shuduo Sang 已提交
4851
static int64_t execInsert(threadInfo *pThreadInfo, uint64_t k)
4852 4853
{
  int affectedRows;
4854
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4855

4856
  verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
S
Shuduo Sang 已提交
4857
            __func__, __LINE__, pThreadInfo->buffer);
4858
  if (superTblInfo) {
4859
    if (superTblInfo->insertMode == TAOSC_IFACE) {
S
Shuduo Sang 已提交
4860 4861 4862
      affectedRows = queryDbExec(
              pThreadInfo->taos,
              pThreadInfo->buffer, INSERT_TYPE, false);
4863
    } else if (superTblInfo->insertMode == REST_IFACE) {
4864
      if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port,
S
Shuduo Sang 已提交
4865
                  pThreadInfo->buffer, NULL /* not set result file */)) {
4866
        affectedRows = -1;
4867
        printf("========restful return fail, threadID[%d]\n",
4868
            pThreadInfo->threadID);
4869 4870 4871
      } else {
        affectedRows = k;
      }
4872
    } else if (superTblInfo->insertMode == STMT_IFACE) {
S
Shuduo Sang 已提交
4873 4874 4875 4876 4877 4878
      debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt);
      if (0 != taos_stmt_execute(pThreadInfo->stmt)) {
        errorPrint("%s() LN%d, failied to execute insert statement\n",
                    __func__, __LINE__);
        exit(-1);
      }
S
Shuduo Sang 已提交
4879 4880

      affectedRows = k;
4881
    } else {
4882
      errorPrint("%s() LN%d: unknown insert mode: %d\n",
4883 4884
        __func__, __LINE__, superTblInfo->insertMode);
      affectedRows = 0;
4885
    }
4886
  } else {
S
Shuduo Sang 已提交
4887
    affectedRows = queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, INSERT_TYPE, false);
4888 4889 4890 4891 4892
  }

  return affectedRows;
}

4893
static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t tableSeq)
4894 4895
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4896
  if (superTblInfo) {
4897
    if (superTblInfo->childTblLimit > 0) {
4898 4899 4900
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName +
            (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
4901
    } else {
4902

4903
        verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n",
4904 4905 4906 4907 4908
                pThreadInfo->threadID, __func__, __LINE__,
                pThreadInfo->start_table_from,
                pThreadInfo->ntables, tableSeq);
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
4909 4910
    }
  } else {
4911
    snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"",
4912
        g_args.tb_prefix, tableSeq);
4913 4914 4915
  }
}

4916 4917 4918 4919 4920
static int64_t generateDataTailWithoutStb(
        uint64_t batch, char* buffer,
        int64_t remainderBufLen, int64_t insertRows,
        uint64_t startFrom, int64_t startTime,
        /* int64_t *pSamplePos, */int64_t *dataLen) {
4921

4922
  uint64_t len = 0;
4923 4924
  char *pstr = buffer;

4925
  verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch);
4926

4927
  int64_t k = 0;
4928
  for (k = 0; k < batch;) {
4929
    char data[MAX_DATA_SIZE];
4930 4931
    memset(data, 0, MAX_DATA_SIZE);

4932
    int64_t retLen = 0;
4933

4934 4935
    char **data_type = g_args.datatype;
    int lenOfBinary = g_args.len_of_binary;
4936

4937 4938 4939 4940 4941 4942 4943 4944 4945
    int64_t randTail = DEFAULT_TIMESTAMP_STEP * k;

    if (g_args.disorderRatio != 0) {
        int rand_num = taosRandom() % 100;
        if (rand_num < g_args.disorderRatio) {
          randTail = (randTail +
                  (taosRandom() % g_args.disorderRange + 1)) * (-1);

          debugPrint("rand data generated, back %"PRId64"\n", randTail);
4946
        }
4947 4948 4949
    } else {
        randTail = DEFAULT_TIMESTAMP_STEP * k;
    }
4950

4951 4952 4953
    retLen = generateData(data, data_type,
            startTime + randTail,
            lenOfBinary);
4954

4955
    if (len > remainderBufLen)
4956
        break;
4957

4958 4959 4960 4961 4962 4963 4964
    pstr += sprintf(pstr, "%s", data);
    k++;
    len += retLen;
    remainderBufLen -= retLen;

    verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n",
            __func__, __LINE__, len, k, buffer);
4965

4966
    startFrom ++;
4967

4968 4969 4970 4971
    if (startFrom >= insertRows) {
      break;
    }
  }
4972

4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013
  *dataLen = len;
  return k;
}

static int64_t generateStbDataTail(
        SSuperTable* superTblInfo,
        uint64_t batch, char* buffer,
        int64_t remainderBufLen, int64_t insertRows,
        uint64_t startFrom, int64_t startTime,
        int64_t *pSamplePos, int64_t *dataLen) {
  uint64_t len = 0;

  char *pstr = buffer;

  verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch);

  int64_t k = 0;
  for (k = 0; k < batch;) {
    char data[MAX_DATA_SIZE];
    memset(data, 0, MAX_DATA_SIZE);

    int64_t retLen = 0;

    if (0 == strncasecmp(superTblInfo->dataSource,
                "sample", strlen("sample"))) {
          retLen = getRowDataFromSample(
                  data,
                  remainderBufLen,
                  startTime + superTblInfo->timeStampStep * k,
                  superTblInfo,
                  pSamplePos);
    } else if (0 == strncasecmp(superTblInfo->dataSource,
                "rand", strlen("rand"))) {
        int64_t randTail = superTblInfo->timeStampStep * k;
        if (superTblInfo->disorderRatio > 0) {
            int rand_num = taosRandom() % 100;
            if(rand_num < superTblInfo->disorderRatio) {
                randTail = (randTail +
                        (taosRandom() % superTblInfo->disorderRange + 1)) * (-1);
                debugPrint("rand data generated, back %"PRId64"\n", randTail);
            }
5014
        }
5015

5016 5017 5018
        int64_t d = startTime + randTail;
        retLen = generateStbRowData(superTblInfo, data, d);
    }
5019

5020
    if (retLen > remainderBufLen) {
5021
        break;
5022 5023
    }

5024 5025 5026 5027 5028
    pstr += snprintf(pstr , retLen + 1, "%s", data);
    k++;
    len += retLen;
    remainderBufLen -= retLen;

5029
    verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n",
5030
            __func__, __LINE__, len, k, buffer);
5031 5032

    startFrom ++;
5033

5034
    if (startFrom >= insertRows) {
5035
      break;
5036
    }
5037
  }
5038

5039 5040 5041
  *dataLen = len;
  return k;
}
5042

5043 5044 5045

static int generateSQLHeadWithoutStb(char *tableName,
        char *dbName,
5046
        char *buffer, int remainderBufLen)
5047 5048
{
  int len;
5049 5050 5051

  char headBuf[HEAD_BUFF_LEN];

5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077
  len = snprintf(
          headBuf,
          HEAD_BUFF_LEN,
          "%s.%s values",
          dbName,
          tableName);

  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

  return len;
}

static int generateStbSQLHead(
        SSuperTable* superTblInfo,
        char *tableName, int32_t tableSeq,
        char *dbName,
        char *buffer, int remainderBufLen)
{
  int len;

  char headBuf[HEAD_BUFF_LEN];

  if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
5078 5079
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
5080
            tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq);
5081
      } else {
5082 5083 5084
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
5085 5086
      }
      if (NULL == tagsValBuf) {
5087
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n",
5088
            __func__, __LINE__);
5089 5090 5091
        return -1;
      }

5092 5093
      len = snprintf(
          headBuf,
5094 5095
                  HEAD_BUFF_LEN,
                  "%s.%s using %s.%s tags %s values",
5096
                  dbName,
5097
                  tableName,
5098
                  dbName,
5099 5100
                  superTblInfo->sTblName,
                  tagsValBuf);
5101 5102
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
5103 5104
      len = snprintf(
          headBuf,
5105 5106
                  HEAD_BUFF_LEN,
                  "%s.%s values",
5107
                  dbName,
5108
                  tableName);
5109
    } else {
5110 5111
      len = snprintf(
          headBuf,
5112 5113
                  HEAD_BUFF_LEN,
                  "%s.%s values",
5114
                  dbName,
5115
                  tableName);
5116 5117
  }

5118 5119 5120 5121 5122
  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

5123 5124 5125
  return len;
}

5126 5127
static int64_t generateStbInterlaceData(
        SSuperTable *superTblInfo,
5128 5129
        char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes,
        uint64_t tableSeq,
5130
        threadInfo *pThreadInfo, char *buffer,
5131
        int64_t insertRows,
5132
        int64_t startTime,
5133
        uint64_t *pRemainderBufLen)
5134
{
5135
  assert(buffer);
5136 5137
  char *pstr = buffer;

5138 5139 5140
  int headLen = generateStbSQLHead(
          superTblInfo,
          tableName, tableSeq, pThreadInfo->db_name,
5141
          pstr, *pRemainderBufLen);
5142 5143 5144 5145 5146

  if (headLen <= 0) {
    return 0;
  }
  // generate data buffer
5147
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n",
5148
            pThreadInfo->threadID, __func__, __LINE__, i, buffer);
5149 5150 5151 5152

  pstr += headLen;
  *pRemainderBufLen -= headLen;

5153
  int64_t dataLen = 0;
5154

5155
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%"PRIu64" batchPerTbl = %"PRIu64"\n",
5156 5157
            pThreadInfo->threadID, __func__, __LINE__,
            i, batchPerTblTimes, batchPerTbl);
5158

5159
  if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
5160
      startTime = taosGetTimestamp(pThreadInfo->time_precision);
5161
  }
5162

5163
  int64_t  k = generateStbDataTail(
5164 5165 5166 5167
            superTblInfo,
            batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
            startTime,
            &(pThreadInfo->samplePos), &dataLen);
5168 5169 5170 5171

  if (k == batchPerTbl) {
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
5172
  } else {
5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207
    debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n",
            __func__, __LINE__, k, batchPerTbl);
    pstr -= headLen;
    pstr[0] = '\0';
    k = 0;
  }

  return k;
}

static int64_t generateInterlaceDataWithoutStb(
        char *tableName, uint64_t batchPerTbl,
        uint64_t tableSeq,
        char *dbName, char *buffer,
        int64_t insertRows,
        uint64_t *pRemainderBufLen)
{
  assert(buffer);
  char *pstr = buffer;

  int headLen = generateSQLHeadWithoutStb(
          tableName, dbName,
            pstr, *pRemainderBufLen);

  if (headLen <= 0) {
    return 0;
  }

  pstr += headLen;
  *pRemainderBufLen -= headLen;

  int64_t dataLen = 0;

  int64_t startTime = 1500000000000;
  int64_t  k = generateDataTailWithoutStb(
5208 5209
            batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
            startTime,
5210
            &dataLen);
5211 5212

  if (k == batchPerTbl) {
5213 5214 5215
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
  } else {
5216
    debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n",
5217
            __func__, __LINE__, k, batchPerTbl);
5218 5219
    pstr -= headLen;
    pstr[0] = '\0';
5220
    k = 0;
5221 5222 5223 5224 5225
  }

  return k;
}

5226 5227
static int64_t generateStbProgressiveData(
        SSuperTable *superTblInfo,
5228
        char *tableName,
5229
        int64_t tableSeq,
5230
        char *dbName, char *buffer,
5231
        int64_t insertRows,
5232
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos,
5233
        int64_t *pRemainderBufLen)
5234 5235
{
  assert(buffer != NULL);
5236
  char *pstr = buffer;
5237

5238
  memset(buffer, 0, *pRemainderBufLen);
5239

5240
  int64_t headLen = generateStbSQLHead(
5241
              superTblInfo,
5242
          tableName, tableSeq, dbName,
5243
          buffer, *pRemainderBufLen);
5244 5245 5246 5247

  if (headLen <= 0) {
    return 0;
  }
5248
  pstr += headLen;
5249
  *pRemainderBufLen -= headLen;
5250

5251
  int64_t dataLen;
5252

5253
  return generateStbDataTail(superTblInfo,
5254 5255
          g_args.num_of_RPR, pstr, *pRemainderBufLen,
          insertRows, startFrom,
5256 5257
          startTime,
          pSamplePos, &dataLen);
5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285
}

static int64_t generateProgressiveDataWithoutStb(
        char *tableName,
        int64_t tableSeq,
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos,
        int64_t *pRemainderBufLen)
{
  assert(buffer != NULL);
  char *pstr = buffer;

  memset(buffer, 0, *pRemainderBufLen);

  int64_t headLen = generateSQLHeadWithoutStb(
          tableName, pThreadInfo->db_name,
          buffer, *pRemainderBufLen);

  if (headLen <= 0) {
    return 0;
  }
  pstr += headLen;
  *pRemainderBufLen -= headLen;

  int64_t dataLen;

  return generateDataTailWithoutStb(
5286 5287 5288
          g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom,
          startTime,
          /*pSamplePos, */&dataLen);
5289
}
5290

5291 5292 5293 5294 5295 5296 5297 5298 5299
static void printStatPerThread(threadInfo *pThreadInfo)
{
  fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n",
          pThreadInfo->threadID,
          pThreadInfo->totalInsertRows,
          pThreadInfo->totalAffectedRows,
          (double)(pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay/1000.0)));
}

5300
// sync write interlace data
5301
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
5302
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
5303 5304
         pThreadInfo->threadID, __func__, __LINE__);

5305
  int64_t insertRows;
5306
  uint64_t interlaceRows;
5307 5308 5309
  uint64_t maxSqlLen;
  int64_t nTimeStampStep;
  uint64_t insert_interval;
5310 5311

  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5312

5313 5314 5315 5316 5317 5318 5319 5320 5321
  if (superTblInfo) {
    insertRows = superTblInfo->insertRows;

    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
5322 5323 5324
    maxSqlLen = superTblInfo->maxSqlLen;
    nTimeStampStep = superTblInfo->timeStampStep;
    insert_interval = superTblInfo->insertInterval;
5325 5326 5327
  } else {
    insertRows = g_args.num_of_DPT;
    interlaceRows = g_args.interlace_rows;
5328 5329 5330
    maxSqlLen = g_args.max_sql_len;
    nTimeStampStep = DEFAULT_TIMESTAMP_STEP;
    insert_interval = g_args.insert_interval;
5331
  }
5332

5333 5334 5335 5336 5337
  debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n",
          pThreadInfo->threadID, __func__, __LINE__,
          pThreadInfo->start_table_from,
          pThreadInfo->ntables, insertRows);

5338
  if (interlaceRows > insertRows)
H
Haojun Liao 已提交
5339
    interlaceRows = insertRows;
5340

5341 5342
  if (interlaceRows > g_args.num_of_RPR)
    interlaceRows = g_args.num_of_RPR;
5343

5344 5345
  uint64_t batchPerTbl = interlaceRows;
  uint64_t batchPerTblTimes;
5346

5347 5348 5349
  if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) {
    batchPerTblTimes =
        g_args.num_of_RPR / interlaceRows;
5350
  } else {
5351
    batchPerTblTimes = 1;
5352 5353
  }

S
Shuduo Sang 已提交
5354 5355
  pThreadInfo->buffer = calloc(maxSqlLen, 1);
  if (NULL == pThreadInfo->buffer) {
5356
    errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5357
              __func__, __LINE__, maxSqlLen, strerror(errno));
5358 5359 5360 5361 5362 5363
    return NULL;
  }

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

5364 5365
  uint64_t st = 0;
  uint64_t et = UINT64_MAX;
5366

5367 5368 5369
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5370

5371
  uint64_t tableSeq = pThreadInfo->start_table_from;
5372 5373
  int64_t startTime = pThreadInfo->start_time;

5374
  uint64_t generatedRecPerTbl = 0;
5375
  bool flagSleep = true;
5376
  uint64_t sleepTimeTotal = 0;
5377

5378
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
5379
    if ((flagSleep) && (insert_interval)) {
5380 5381
        st = taosGetTimestampMs();
        flagSleep = false;
5382 5383
    }
    // generate data
S
Shuduo Sang 已提交
5384
    memset(pThreadInfo->buffer, 0, maxSqlLen);
5385
    uint64_t remainderBufLen = maxSqlLen;
5386

S
Shuduo Sang 已提交
5387
    char *pstr = pThreadInfo->buffer;
5388

5389 5390
    int len = snprintf(pstr,
            strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO);
5391 5392 5393
    pstr += len;
    remainderBufLen -= len;

5394
    uint64_t recOfBatch = 0;
5395

5396
    for (uint64_t i = 0; i < batchPerTblTimes; i ++) {
5397 5398
        char tableName[TSDB_TABLE_NAME_LEN];

5399
      getTableName(tableName, pThreadInfo, tableSeq);
5400 5401
      if (0 == strlen(tableName)) {
        errorPrint("[%d] %s() LN%d, getTableName return null\n",
5402
            pThreadInfo->threadID, __func__, __LINE__);
S
Shuduo Sang 已提交
5403
        free(pThreadInfo->buffer);
5404 5405
        return NULL;
      }
5406

5407
      uint64_t oldRemainderLen = remainderBufLen;
5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426

      int64_t generated;
      if (superTblInfo) {
        generated = generateStbInterlaceData(
                superTblInfo,
                tableName, batchPerTbl, i, batchPerTblTimes,
                tableSeq,
                pThreadInfo, pstr,
                insertRows,
                startTime,
                &remainderBufLen);
      } else {
        generated = generateInterlaceDataWithoutStb(
                tableName, batchPerTbl,
                tableSeq,
                pThreadInfo->db_name, pstr,
                insertRows,
                &remainderBufLen);
      }
5427

5428 5429
      debugPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5430
      if (generated < 0) {
5431
        errorPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
5432
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5433
        goto free_of_interlace;
5434 5435
      } else if (generated == 0) {
        break;
5436 5437
      }

5438
      tableSeq ++;
5439
      recOfBatch += batchPerTbl;
5440
      pstr += (oldRemainderLen - remainderBufLen);
5441
//      startTime += batchPerTbl * superTblInfo->timeStampStep;
5442
      pThreadInfo->totalInsertRows += batchPerTbl;
5443
      verbosePrint("[%d] %s() LN%d batchPerTbl=%"PRId64" recOfBatch=%"PRId64"\n",
5444 5445
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
5446

5447
      if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
5448 5449 5450
            // turn to first table
            tableSeq = pThreadInfo->start_table_from;
            generatedRecPerTbl += batchPerTbl;
5451

5452 5453
            startTime = pThreadInfo->start_time
              + generatedRecPerTbl * nTimeStampStep;
5454

5455 5456 5457
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;
5458

5459 5460 5461
            int remainRows = insertRows - generatedRecPerTbl;
            if ((remainRows > 0) && (batchPerTbl > remainRows))
              batchPerTbl = remainRows;
5462

5463 5464 5465
            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
      }
5466

5467
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n",
5468 5469
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);
5470 5471

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
5472 5473 5474
        break;
    }

5475
    verbosePrint("[%d] %s() LN%d recOfBatch=%"PRIu64" totalInsertRows=%"PRIu64"\n",
5476 5477
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
5478
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
S
Shuduo Sang 已提交
5479
           pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer);
5480

5481
    startTs = taosGetTimestampMs();
5482

5483 5484 5485 5486 5487 5488 5489
    if (recOfBatch == 0) {
      errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n",
              pThreadInfo->threadID, __func__, __LINE__,
              recOfBatch);
      errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n");
      goto free_of_interlace;
    }
S
Shuduo Sang 已提交
5490
    int64_t affectedRows = execInsert(pThreadInfo, recOfBatch);
5491

5492
    endTs = taosGetTimestampMs();
5493 5494
    uint64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n",
5495
            __func__, __LINE__, delay);
5496 5497 5498
    verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5499 5500 5501 5502 5503 5504

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

5505 5506
    if (recOfBatch != affectedRows) {
        errorPrint("[%d] %s() LN%d execInsert insert %"PRIu64", affected rows: %"PRId64"\n%s\n",
5507
                pThreadInfo->threadID, __func__, __LINE__,
S
Shuduo Sang 已提交
5508
                recOfBatch, affectedRows, pThreadInfo->buffer);
5509
        goto free_of_interlace;
5510
    }
5511

5512
    pThreadInfo->totalAffectedRows += affectedRows;
5513

5514 5515
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
5516
      printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n",
5517 5518 5519
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5520 5521
      lastPrintTime = currentPrintTime;
    }
5522

5523
    if ((insert_interval) && flagSleep) {
5524
      et = taosGetTimestampMs();
5525

5526
      if (insert_interval > (et - st) ) {
S
Shuduo Sang 已提交
5527 5528
        uint64_t sleepTime = insert_interval - (et -st);
        performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n",
5529
                    __func__, __LINE__, sleepTime);
5530 5531 5532
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
5533
    }
5534 5535
  }

5536
free_of_interlace:
S
Shuduo Sang 已提交
5537
  tmfree(pThreadInfo->buffer);
5538
  printStatPerThread(pThreadInfo);
5539 5540 5541
  return NULL;
}

5542
// sync insertion progressive data
5543
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
5544
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
5545

5546
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5547
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
5548 5549 5550 5551 5552 5553
  int64_t timeStampStep =
      superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
  int64_t insertRows =
        (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
  verbosePrint("%s() LN%d insertRows=%"PRId64"\n",
            __func__, __LINE__, insertRows);
5554

S
Shuduo Sang 已提交
5555 5556
  pThreadInfo->buffer = calloc(maxSqlLen, 1);
  if (NULL == pThreadInfo->buffer) {
5557
    errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5558 5559
              maxSqlLen,
              strerror(errno));
5560 5561
    return NULL;
  }
5562

5563 5564 5565
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5566

5567
/*  int insert_interval =
5568
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5569 5570
  uint64_t st = 0;
  uint64_t et = 0xffffffff;
5571
  */
5572

5573 5574
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
5575

5576
  pThreadInfo->samplePos = 0;
5577

5578 5579 5580
  for (uint64_t tableSeq = pThreadInfo->start_table_from;
          tableSeq <= pThreadInfo->end_table_to;
          tableSeq ++) {
5581
    int64_t start_time = pThreadInfo->start_time;
5582

5583
    for (uint64_t i = 0; i < insertRows;) {
5584 5585
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
5586
      verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n",
5587 5588
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);
5589

5590
      int64_t remainderBufLen = maxSqlLen;
S
Shuduo Sang 已提交
5591
      char *pstr = pThreadInfo->buffer;
5592

5593 5594
      int len = snprintf(pstr,
              strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO);
5595 5596 5597 5598

      pstr += len;
      remainderBufLen -= len;

5599 5600 5601 5602 5603 5604 5605 5606 5607 5608
      int64_t generated;
      if (superTblInfo) {
        generated = generateStbProgressiveData(
                superTblInfo,
              tableName, tableSeq, pThreadInfo->db_name, pstr, insertRows,
            i, start_time,
            &(pThreadInfo->samplePos),
            &remainderBufLen);
      } else {
        generated = generateProgressiveDataWithoutStb(
5609 5610 5611 5612
              tableName, tableSeq, pThreadInfo, pstr, insertRows,
            i, start_time,
            &(pThreadInfo->samplePos),
            &remainderBufLen);
5613
      }
5614 5615 5616
      if (generated > 0)
        i += generated;
      else
5617
        goto free_of_progressive;
5618

5619
      start_time +=  generated * timeStampStep;
5620
      pThreadInfo->totalInsertRows += generated;
5621

5622
      startTs = taosGetTimestampMs();
5623

S
Shuduo Sang 已提交
5624
      int64_t affectedRows = execInsert(pThreadInfo, generated);
5625

5626
      endTs = taosGetTimestampMs();
5627
      uint64_t delay = endTs - startTs;
5628
      performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n",
5629
              __func__, __LINE__, delay);
5630 5631 5632
      verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5633

5634 5635 5636 5637
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
5638

5639 5640 5641 5642 5643
      if (affectedRows < 0) {
        errorPrint("%s() LN%d, affected rows: %"PRId64"\n",
                __func__, __LINE__, affectedRows);
        goto free_of_progressive;
      }
5644 5645 5646

      pThreadInfo->totalAffectedRows += affectedRows;

5647 5648
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
5649
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
5650 5651 5652
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5653 5654 5655
        lastPrintTime = currentPrintTime;
      }

5656
      if (i >= insertRows)
5657 5658 5659
        break;
    }   // num_of_DPT

5660 5661
    if ((g_args.verbose_print) &&
      (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) &&
5662 5663
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
5664
          verbosePrint("%s() LN%d samplePos=%"PRId64"\n",
5665
                  __func__, __LINE__, pThreadInfo->samplePos);
5666
    }
5667
  } // tableSeq
5668

5669
free_of_progressive:
S
Shuduo Sang 已提交
5670
  tmfree(pThreadInfo->buffer);
5671
  printStatPerThread(pThreadInfo);
5672
  return NULL;
H
Hui Li 已提交
5673 5674
}

5675 5676
static void* syncWrite(void *sarg) {

5677 5678
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5679

5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691
  int interlaceRows;

  if (superTblInfo) {
    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    interlaceRows = g_args.interlace_rows;
  }
5692

5693
  if (interlaceRows > 0) {
5694
    // interlace mode
5695
    return syncWriteInterlace(pThreadInfo);
5696 5697
  } else {
    // progressive mode
5698
    return syncWriteProgressive(pThreadInfo);
5699 5700 5701
  }
}

5702
static void callBack(void *param, TAOS_RES *res, int code) {
5703 5704
  threadInfo* pThreadInfo = (threadInfo*)param;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5705

5706 5707
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5708
  if (insert_interval) {
5709 5710 5711
    pThreadInfo->et = taosGetTimestampMs();
    if ((pThreadInfo->et - pThreadInfo->st) < insert_interval) {
      taosMsleep(insert_interval - (pThreadInfo->et - pThreadInfo->st)); // ms
5712
    }
H
Hui Li 已提交
5713
  }
5714

5715
  char *buffer = calloc(1, pThreadInfo->superTblInfo->maxSqlLen);
5716
  char data[MAX_DATA_SIZE];
5717
  char *pstr = buffer;
5718 5719
  pstr += sprintf(pstr, "insert into %s.%s%"PRId64" values",
          pThreadInfo->db_name, pThreadInfo->tb_prefix,
5720
          pThreadInfo->start_table_from);
5721 5722 5723 5724 5725 5726 5727
//  if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
  if (pThreadInfo->counter >= g_args.num_of_RPR) {
    pThreadInfo->start_table_from++;
    pThreadInfo->counter = 0;
  }
  if (pThreadInfo->start_table_from > pThreadInfo->end_table_to) {
    tsem_post(&pThreadInfo->lock_sem);
5728 5729 5730 5731
    free(buffer);
    taos_free_result(res);
    return;
  }
5732

5733
  for (int i = 0; i < g_args.num_of_RPR; i++) {
5734
    int rand_num = taosRandom() % 100;
5735
    if (0 != pThreadInfo->superTblInfo->disorderRatio
5736
            && rand_num < pThreadInfo->superTblInfo->disorderRatio) {
5737 5738 5739
      int64_t d = pThreadInfo->lastTs
          - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1);
      generateStbRowData(pThreadInfo->superTblInfo, data, d);
5740
    } else {
5741 5742
      generateStbRowData(pThreadInfo->superTblInfo,
              data, pThreadInfo->lastTs += 1000);
H
Hui Li 已提交
5743
    }
5744
    pstr += sprintf(pstr, "%s", data);
5745
    pThreadInfo->counter++;
H
Hui Li 已提交
5746

5747
    if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
5748
      break;
H
Hui Li 已提交
5749 5750
    }
  }
5751

5752
  if (insert_interval) {
5753
    pThreadInfo->st = taosGetTimestampMs();
5754
  }
5755
  taos_query_a(pThreadInfo->taos, buffer, callBack, pThreadInfo);
5756
  free(buffer);
H
Hui Li 已提交
5757

5758
  taos_free_result(res);
H
Hui Li 已提交
5759 5760
}

5761
static void *asyncWrite(void *sarg) {
5762 5763
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5764

5765 5766 5767
  pThreadInfo->st = 0;
  pThreadInfo->et = 0;
  pThreadInfo->lastTs = pThreadInfo->start_time;
5768

5769
  int insert_interval =
5770
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5771
  if (insert_interval) {
5772
    pThreadInfo->st = taosGetTimestampMs();
H
Hui Li 已提交
5773
  }
5774
  taos_query_a(pThreadInfo->taos, "show databases", callBack, pThreadInfo);
5775

5776
  tsem_wait(&(pThreadInfo->lock_sem));
H
Hui Li 已提交
5777 5778 5779 5780

  return NULL;
}

5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806
static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in *serv_addr)
{
  uint16_t rest_port = port + TSDB_PORT_HTTP;
  struct hostent *server = gethostbyname(host);
  if ((server == NULL) || (server->h_addr == NULL)) {
    errorPrint("%s", "ERROR, no such host");
    return -1;
  }

  debugPrint("h_name: %s\nh_addr=%p\nh_addretype: %s\nh_length: %d\n",
            server->h_name,
            server->h_addr,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

  memset(serv_addr, 0, sizeof(struct sockaddr_in));
  serv_addr->sin_family = AF_INET;
  serv_addr->sin_port = htons(rest_port);
#ifdef WINDOWS
  serv_addr->sin_addr.s_addr = inet_addr(host);
#else
  memcpy(&(serv_addr->sin_addr.s_addr), server->h_addr, server->h_length);
#endif
  return 0;
}

5807
static void startMultiThreadInsertData(int threads, char* db_name,
5808
        char* precision,SSuperTable* superTblInfo) {
5809

5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825
  //TAOS* taos;
  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
  //  if (NULL == taos) {
  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
  //    exit(-1);
  //  }
  //}

  int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
  if (0 != precision[0]) {
    if (0 == strncasecmp(precision, "ms", 2)) {
      timePrec = TSDB_TIME_PRECISION_MILLI;
    }  else if (0 == strncasecmp(precision, "us", 2)) {
      timePrec = TSDB_TIME_PRECISION_MICRO;
    }  else {
5826
      errorPrint("Not support precision: %s\n", precision);
5827 5828 5829 5830
      exit(-1);
    }
  }

5831
  int64_t start_time;
5832 5833
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
5834
        start_time = taosGetTimestamp(timePrec);
5835
    } else {
5836
      if (TSDB_CODE_SUCCESS != taosParseTime(
5837 5838 5839 5840 5841
        superTblInfo->startTimestamp,
        &start_time,
        strlen(superTblInfo->startTimestamp),
        timePrec, 0)) {
          ERROR_EXIT("failed to parse time!\n");
5842
      }
5843
    }
5844
  } else {
5845
     start_time = 1500000000000;
5846 5847
  }

5848
  int64_t start = taosGetTimestampMs();
5849

5850
  // read sample data from file first
5851
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5852
              "sample", strlen("sample")))) {
5853
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5854
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
5855
              __func__, __LINE__);
5856 5857 5858 5859
      exit(-1);
    }
  }

5860
  // read sample data from file first
5861
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5862
              "sample", strlen("sample")))) {
5863
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5864
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
5865
              __func__, __LINE__);
5866 5867 5868 5869
      exit(-1);
    }
  }

5870
  TAOS* taos0 = taos_connect(
5871 5872
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
5873
  if (NULL == taos0) {
5874
    errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
5875
                __func__, __LINE__, taos_errstr(NULL));
5876 5877 5878
    exit(-1);
  }

S
Shuduo Sang 已提交
5879
  int64_t ntables = 0;
S
Shuduo Sang 已提交
5880
  uint64_t startFrom;
5881

5882
  if (superTblInfo) {
5883 5884
    int64_t limit;
    uint64_t offset;
5885

H
Haojun Liao 已提交
5886
    if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) &&
5887
            ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) {
H
Haojun Liao 已提交
5888
      printf("WARNING: offset and limit will not be used since the child tables not exists!\n");
5889 5890
    }

5891
    if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) {
5892 5893
      if ((superTblInfo->childTblLimit < 0)
          || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit)
5894
            > (superTblInfo->childTblCount))) {
5895 5896 5897 5898 5899 5900
        superTblInfo->childTblLimit =
            superTblInfo->childTblCount - superTblInfo->childTblOffset;
      }

      offset = superTblInfo->childTblOffset;
      limit = superTblInfo->childTblLimit;
5901
    } else {
5902 5903
      limit = superTblInfo->childTblCount;
      offset = 0;
5904 5905
    }

5906 5907 5908
    ntables = limit;
    startFrom = offset;

5909 5910 5911 5912
    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
        && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
            > superTblInfo->childTblCount)) {
      printf("WARNING: specified offset + limit > child table count!\n");
5913
      prompt();
5914 5915 5916
    }

    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
5917
            && (0 == superTblInfo->childTblLimit)) {
5918
      printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
5919
      prompt();
5920 5921
    }

5922
    superTblInfo->childTblName = (char*)calloc(1,
5923
        limit * TSDB_TABLE_NAME_LEN);
5924
    if (superTblInfo->childTblName == NULL) {
5925
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
5926
      taos_close(taos0);
5927 5928 5929
      exit(-1);
    }

5930
    int64_t childTblCount;
5931
    getChildNameOfSuperTableWithLimitAndOffset(
5932
        taos0,
5933 5934 5935 5936
        db_name, superTblInfo->sTblName,
        &superTblInfo->childTblName, &childTblCount,
        limit,
        offset);
5937 5938 5939
  } else {
    ntables = g_args.num_of_tables;
    startFrom = 0;
5940
  }
5941

5942
  taos_close(taos0);
5943

S
Shuduo Sang 已提交
5944
  int64_t a = ntables / threads;
5945 5946 5947 5948 5949
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

S
Shuduo Sang 已提交
5950
  int64_t b = 0;
5951 5952 5953 5954
  if (threads != 0) {
    b = ntables % threads;
  }

5955
  if ((superTblInfo)
5956 5957 5958 5959
      && (superTblInfo->insertMode == REST_IFACE)) {
      if (convertHostToServAddr(g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) {
        exit(-1);
      }
5960 5961
  }

5962 5963 5964 5965 5966 5967 5968 5969 5970
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  assert(pids != NULL);

  threadInfo *infos = calloc(1, threads * sizeof(threadInfo));
  assert(infos != NULL);

  memset(pids, 0, threads * sizeof(pthread_t));
  memset(infos, 0, threads * sizeof(threadInfo));

H
Hui Li 已提交
5971
  for (int i = 0; i < threads; i++) {
5972
    threadInfo *t_info = infos + i;
H
Hui Li 已提交
5973 5974
    t_info->threadID = i;
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
5975
    t_info->time_precision = timePrec;
5976 5977 5978
    t_info->superTblInfo = superTblInfo;

    t_info->start_time = start_time;
5979
    t_info->minDelay = UINT64_MAX;
5980

5981
    if ((NULL == superTblInfo) ||
5982
            (superTblInfo->insertMode != REST_IFACE)) {
5983
      //t_info->taos = taos;
5984
      t_info->taos = taos_connect(
5985 5986
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
5987
      if (NULL == t_info->taos) {
5988
        errorPrint(
5989 5990
                "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n",
                __func__, __LINE__,
5991
                taos_errstr(NULL));
5992
        free(infos);
5993 5994
        exit(-1);
      }
5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007

      if ((superTblInfo) && (superTblInfo->insertMode == STMT_IFACE)) {
        t_info->stmt = taos_stmt_init(t_info->taos);
        if (NULL == t_info->stmt) {
            errorPrint(
                "%s() LN%d, failed init stmt, reason: %s\n",
                __func__, __LINE__,
                taos_errstr(NULL));
            free(pids);
            free(infos);
            exit(-1);
        }
      }
6008 6009 6010 6011
    } else {
      t_info->taos = NULL;
    }

6012
/*    if ((NULL == superTblInfo)
6013
            || (0 == superTblInfo->multiThreadWriteOneTbl)) {
6014
            */
6015 6016 6017 6018
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
6019
/*    } else {
6020 6021
      t_info->start_table_from = 0;
      t_info->ntables = superTblInfo->childTblCount;
6022 6023
      t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint();
    }
6024
*/
6025
    tsem_init(&(t_info->lock_sem), 0, 0);
6026
    if (ASYNC_MODE == g_Dbs.asyncMode) {
6027
      pthread_create(pids + i, NULL, asyncWrite, t_info);
6028 6029
    } else {
      pthread_create(pids + i, NULL, syncWrite, t_info);
6030
    }
H
Hui Li 已提交
6031
  }
6032

H
Hui Li 已提交
6033 6034 6035 6036
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

6037 6038 6039 6040
  uint64_t totalDelay = 0;
  uint64_t maxDelay = 0;
  uint64_t minDelay = UINT64_MAX;
  uint64_t cntDelay = 1;
6041 6042
  double  avgDelay = 0;

H
Hui Li 已提交
6043
  for (int i = 0; i < threads; i++) {
6044 6045
    threadInfo *t_info = infos + i;

S
TD-1057  
Shengliang Guan 已提交
6046
    tsem_destroy(&(t_info->lock_sem));
6047 6048 6049 6050

    if (t_info->stmt) {
      taos_stmt_close(t_info->stmt);
    }
6051 6052
    taos_close(t_info->taos);

6053
    debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n",
6054 6055 6056
            __func__, __LINE__,
            t_info->threadID, t_info->totalInsertRows,
            t_info->totalAffectedRows);
6057
    if (superTblInfo) {
6058 6059
        superTblInfo->totalAffectedRows += t_info->totalAffectedRows;
        superTblInfo->totalInsertRows += t_info->totalInsertRows;
6060
    } else {
6061 6062
        g_args.totalAffectedRows += t_info->totalAffectedRows;
        g_args.totalInsertRows += t_info->totalInsertRows;
6063
    }
6064 6065 6066 6067

    totalDelay  += t_info->totalDelay;
    cntDelay   += t_info->cntDelay;
    if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay;
6068
    if (t_info->minDelay < minDelay) minDelay = t_info->minDelay;
H
Hui Li 已提交
6069
  }
6070
  cntDelay -= 1;
H
Hui Li 已提交
6071

6072
  if (cntDelay == 0)    cntDelay = 1;
6073 6074
  avgDelay = (double)totalDelay / cntDelay;

6075 6076
  int64_t end = taosGetTimestampMs();
  int64_t t = end - start;
6077 6078

  if (superTblInfo) {
6079
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
6080 6081 6082 6083
          t / 1000.0, superTblInfo->totalInsertRows,
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
6084 6085 6086 6087

    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
6088 6089 6090 6091
          t / 1000.0, superTblInfo->totalInsertRows,
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
6092
    }
6093
  } else {
6094
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
6095 6096 6097 6098
          t / 1000.0, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          (double)g_args.totalInsertRows / (t / 1000.0));
6099 6100 6101
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
6102 6103 6104 6105
          t * 1000.0, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          (double)g_args.totalInsertRows / (t / 1000.0));
6106
    }
6107
  }
6108

6109
  fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
6110
          avgDelay, maxDelay, minDelay);
6111 6112
  if (g_fpOfInsertResult) {
    fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
6113
          avgDelay, maxDelay, minDelay);
6114
  }
6115

6116 6117
  //taos_close(taos);

H
Hui Li 已提交
6118
  free(pids);
6119
  free(infos);
H
Hui Li 已提交
6120 6121
}

6122
static void *readTable(void *sarg) {
6123
#if 1
6124
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
6125 6126
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
6127
  uint64_t sTime = rinfo->start_time;
H
hzcheng 已提交
6128 6129
  char *tb_prefix = rinfo->tb_prefix;
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
6130
  if (NULL == fp) {
6131
    errorPrint( "fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
H
Hui Li 已提交
6132 6133
    return NULL;
  }
6134

6135
  int64_t num_of_DPT;
6136
/*  if (rinfo->superTblInfo) {
6137 6138
    num_of_DPT = rinfo->superTblInfo->insertRows; //  nrecords_per_table;
  } else {
6139
  */
6140
      num_of_DPT = g_args.num_of_DPT;
6141
//  }
6142

S
Shuduo Sang 已提交
6143 6144
  int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
  int64_t totalData = num_of_DPT * num_of_tables;
6145
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6146 6147 6148 6149 6150

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
S
Shuduo Sang 已提交
6151
  printf("%"PRId64" records:\n", totalData);
H
hzcheng 已提交
6152 6153
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

6154
  for (int j = 0; j < n; j++) {
H
hzcheng 已提交
6155
    double totalT = 0;
6156
    uint64_t count = 0;
6157 6158
    for (int64_t i = 0; i < num_of_tables; i++) {
      sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64,
6159
              aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
6160

6161
      double t = taosGetTimestampMs();
S
Shuaiqiang Chang 已提交
6162 6163
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
6164

S
Shuaiqiang Chang 已提交
6165
      if (code != 0) {
6166
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6167
        taos_free_result(pSql);
H
hzcheng 已提交
6168
        taos_close(taos);
6169
        fclose(fp);
6170
        return NULL;
H
hzcheng 已提交
6171 6172
      }

6173
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6174 6175 6176
        count++;
      }

6177
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6178 6179
      totalT += t;

S
Shuaiqiang Chang 已提交
6180
      taos_free_result(pSql);
H
hzcheng 已提交
6181 6182
    }

6183
    fprintf(fp, "|%10s  |   %"PRId64"   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
6184
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
6185
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
6186
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000);
H
hzcheng 已提交
6187 6188 6189
  }
  fprintf(fp, "\n");
  fclose(fp);
6190
#endif
H
hzcheng 已提交
6191 6192 6193
  return NULL;
}

6194
static void *readMetric(void *sarg) {
6195
#if 1
6196
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
6197 6198 6199
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
6200 6201 6202 6203
  if (NULL == fp) {
    printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
    return NULL;
  }
6204

6205
  int64_t num_of_DPT = rinfo->superTblInfo->insertRows;
S
Shuduo Sang 已提交
6206 6207
  int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
  int64_t totalData = num_of_DPT * num_of_tables;
6208
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6209 6210 6211 6212 6213

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
S
Shuduo Sang 已提交
6214 6215
  printf("%"PRId64" records:\n", totalData);
  fprintf(fp, "Querying On %"PRId64" records:\n", totalData);
H
hzcheng 已提交
6216 6217

  for (int j = 0; j < n; j++) {
6218
    char condition[COND_BUF_LEN] = "\0";
B
Bomin Zhang 已提交
6219
    char tempS[64] = "\0";
H
hzcheng 已提交
6220

6221
    int64_t m = 10 < num_of_tables ? 10 : num_of_tables;
H
hzcheng 已提交
6222

6223
    for (int64_t i = 1; i <= m; i++) {
H
hzcheng 已提交
6224
      if (i == 1) {
6225
        sprintf(tempS, "t1 = %"PRId64"", i);
H
hzcheng 已提交
6226
      } else {
6227
        sprintf(tempS, " or t1 = %"PRId64" ", i);
H
hzcheng 已提交
6228
      }
6229
      strncat(condition, tempS, COND_BUF_LEN - 1);
H
hzcheng 已提交
6230

L
liu0x54 已提交
6231
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
6232 6233 6234 6235

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

6236
      double t = taosGetTimestampMs();
H
hzcheng 已提交
6237

S
Shuaiqiang Chang 已提交
6238 6239 6240 6241
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
6242
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6243
        taos_free_result(pSql);
H
hzcheng 已提交
6244
        taos_close(taos);
6245
        fclose(fp);
6246
        return NULL;
H
hzcheng 已提交
6247 6248
      }
      int count = 0;
6249
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6250 6251
        count++;
      }
6252
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6253

6254
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
6255 6256
              num_of_tables * num_of_DPT / (t * 1000.0), t);
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0);
H
hzcheng 已提交
6257

S
Shuaiqiang Chang 已提交
6258
      taos_free_result(pSql);
H
hzcheng 已提交
6259 6260 6261 6262
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
6263
#endif
H
hzcheng 已提交
6264 6265 6266
  return NULL;
}

6267 6268 6269 6270 6271 6272 6273
static void prompt()
{
  if (!g_args.answer_yes) {
    printf("         Press enter key to continue or Ctrl-C to stop\n\n");
    (void)getchar();
  }
}
H
Hui Li 已提交
6274

6275
static int insertTestProcess() {
6276

6277 6278 6279
  setupForAnsiEscape();
  int ret = printfInsertMeta();
  resetAfterAnsiEscape();
6280

6281 6282 6283
  if (ret == -1)
    exit(EXIT_FAILURE);

6284
  debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
6285 6286
  g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
  if (NULL == g_fpOfInsertResult) {
6287
    errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
6288 6289
    return -1;
  }
6290

6291 6292
  if (g_fpOfInsertResult)
    printfInsertMetaToFile(g_fpOfInsertResult);
6293

6294
  prompt();
6295

6296 6297 6298
  init_rand_data();

  // create database and super tables
6299
  if(createDatabasesAndStables() != 0) {
6300 6301
    if (g_fpOfInsertResult)
      fclose(g_fpOfInsertResult);
6302 6303
    return -1;
  }
6304 6305

  // pretreatement
6306
  prepareSampleData();
6307

6308 6309 6310 6311
  double start;
  double end;

  // create child tables
6312
  start = taosGetTimestampMs();
6313
  createChildTables();
6314
  end = taosGetTimestampMs();
6315

6316
  if (g_totalChildTables > 0) {
6317
    fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6318
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6319 6320
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
6321
            "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6322
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6323
    }
6324
  }
6325

6326
  taosMsleep(1000);
6327
  // create sub threads for inserting data
6328
  //start = taosGetTimestampMs();
6329
  for (int i = 0; i < g_Dbs.dbCount; i++) {
6330 6331
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
S
Shuduo Sang 已提交
6332
        for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6333 6334 6335 6336 6337

          SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];

          if (superTblInfo && (superTblInfo->insertRows > 0)) {
            startMultiThreadInsertData(
6338 6339 6340 6341
              g_Dbs.threadCount,
              g_Dbs.db[i].dbName,
              g_Dbs.db[i].dbCfg.precision,
              superTblInfo);
6342
          }
6343
        }
6344
      }
6345
    } else {
6346
        startMultiThreadInsertData(
6347 6348 6349
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
6350
          NULL);
H
Hui Li 已提交
6351
    }
6352
  }
6353
  //end = taosGetTimestampMs();
6354

6355
  //int64_t    totalInsertRows = 0;
6356
  //int64_t    totalAffectedRows = 0;
6357
  //for (int i = 0; i < g_Dbs.dbCount; i++) {
6358
  //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6359
  //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
6360 6361
  //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
  //}
6362
  //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
6363
  postFreeResource();
6364

6365 6366 6367
  return 0;
}

6368 6369
static void *specifiedTableQuery(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6370

6371
  if (pThreadInfo->taos == NULL) {
6372 6373
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
6374 6375 6376 6377
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
6378 6379
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6380
            pThreadInfo->threadID, taos_errstr(NULL));
6381 6382
      return NULL;
    } else {
6383
      pThreadInfo->taos = taos;
6384 6385 6386
    }
  }

6387 6388
  char sqlStr[MAX_DB_NAME_SIZE + 5];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6389 6390
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6391 6392 6393 6394
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
    return NULL;
  }
6395

6396 6397
  uint64_t st = 0;
  uint64_t et = 0;
6398

6399
  uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes;
6400

6401 6402 6403
  uint64_t totalQueried = 0;
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
6404

6405 6406 6407 6408 6409 6410
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
        sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }

6411
  while(queryTimes --) {
6412
    if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) <
6413
            (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) {
6414
      taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms
6415 6416
    }

6417 6418 6419
    st = taosGetTimestampMs();

    selectAndGetResult(pThreadInfo,
6420
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]);
6421

6422 6423 6424 6425
    et = taosGetTimestampMs();
    printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n",
              taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0);

6426 6427
    totalQueried ++;
    g_queryInfo.specifiedQueryInfo.totalQueried ++;
6428

6429 6430
    uint64_t  currentPrintTime = taosGetTimestampMs();
    uint64_t  endTs = taosGetTimestampMs();
6431
    if (currentPrintTime - lastPrintTime > 30*1000) {
6432
      debugPrint("%s() LN%d, endTs=%"PRIu64"ms, startTs=%"PRIu64"ms\n",
6433
          __func__, __LINE__, endTs, startTs);
6434
      printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.6f\n",
6435 6436
                    pThreadInfo->threadID,
                    totalQueried,
6437
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6438
      lastPrintTime = currentPrintTime;
6439
    }
H
Hui Li 已提交
6440
  }
6441 6442
  return NULL;
}
H
Hui Li 已提交
6443

6444
static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) {
6445 6446
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
6447 6448
  sprintf(subTblName, "%s.%s",
          g_queryInfo.dbName,
6449
          g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
6450 6451

  //printf("inSql: %s\n", inSql);
6452

6453 6454
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
6455
    return;
H
Hui Li 已提交
6456
  }
6457

6458
  tstrncpy(outSql, inSql, pos - inSql + 1);
6459
  //printf("1: %s\n", outSql);
6460
  strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1);
6461
  //printf("2: %s\n", outSql);
6462
  strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1);
6463
  //printf("3: %s\n", outSql);
H
Hui Li 已提交
6464 6465
}

6466
static void *superTableQuery(void *sarg) {
6467
  char sqlstr[MAX_QUERY_SQL_LENGTH];
6468
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6469

6470
  if (pThreadInfo->taos == NULL) {
6471 6472 6473 6474 6475 6476 6477 6478
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6479
            pThreadInfo->threadID, taos_errstr(NULL));
6480 6481
      return NULL;
    } else {
6482
      pThreadInfo->taos = taos;
6483 6484 6485
    }
  }

6486 6487
  uint64_t st = 0;
  uint64_t et = (int64_t)g_queryInfo.superQueryInfo.queryInterval;
6488

6489 6490 6491
  uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes;
  uint64_t totalQueried = 0;
  uint64_t  startTs = taosGetTimestampMs();
6492

6493
  uint64_t  lastPrintTime = taosGetTimestampMs();
6494
  while(queryTimes --) {
6495 6496 6497
    if (g_queryInfo.superQueryInfo.queryInterval
            && (et - st) < (int64_t)g_queryInfo.superQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.superQueryInfo.queryInterval - (et - st)); // ms
6498
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to);
6499
    }
H
Hui Li 已提交
6500

6501
    st = taosGetTimestampMs();
6502
    for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) {
6503
      for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
6504
        memset(sqlstr,0,sizeof(sqlstr));
6505
        replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i);
6506
        if (g_queryInfo.superQueryInfo.result[j][0] != 0) {
6507
          sprintf(pThreadInfo->fp, "%s-%d",
6508
                  g_queryInfo.superQueryInfo.result[j],
6509
                  pThreadInfo->threadID);
6510
        }
6511
        selectAndGetResult(pThreadInfo, sqlstr);
6512 6513 6514 6515 6516 6517 6518

        totalQueried++;
        g_queryInfo.superQueryInfo.totalQueried ++;

        int64_t  currentPrintTime = taosGetTimestampMs();
        int64_t  endTs = taosGetTimestampMs();
        if (currentPrintTime - lastPrintTime > 30*1000) {
6519
          printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.3f\n",
6520 6521
                    pThreadInfo->threadID,
                    totalQueried,
6522
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6523
          lastPrintTime = currentPrintTime;
6524
        }
6525
      }
H
Hui Li 已提交
6526
    }
6527
    et = taosGetTimestampMs();
6528
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%"PRIu64" - %"PRIu64"] once queries duration:%.4fs\n\n",
6529
            taosGetSelfPthreadId(),
6530 6531
            pThreadInfo->start_table_from,
            pThreadInfo->end_table_to,
6532
            (double)(et - st)/1000.0);
6533
  }
6534

6535 6536 6537
  return NULL;
}

6538
static int queryTestProcess() {
6539 6540 6541 6542

  setupForAnsiEscape();
  printfQueryMeta();
  resetAfterAnsiEscape();
6543 6544 6545 6546 6547 6548

  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
6549
          g_queryInfo.port);
6550
  if (taos == NULL) {
6551 6552
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6553 6554 6555
    exit(-1);
  }

6556
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6557 6558
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6559 6560 6561
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6562
  }
6563

6564
  prompt();
6565

6566 6567 6568
  if (g_args.debug_print || g_args.verbose_print) {
    printfQuerySystemInfo(taos);
  }
6569

6570 6571 6572 6573 6574 6575
  if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
    if (convertHostToServAddr(
        g_queryInfo.host, g_queryInfo.port, &g_queryInfo.serv_addr) != 0)
      exit(-1);
  }

6576 6577 6578
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
6579
  int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent;
6580
  uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount;
6581

6582
  uint64_t startTs = taosGetTimestampMs();
6583

6584 6585 6586 6587 6588 6589
  if ((nSqlCount > 0) && (nConcurrent > 0)) {

    pids  = malloc(nConcurrent * nSqlCount * sizeof(pthread_t));
    infos = malloc(nConcurrent * nSqlCount * sizeof(threadInfo));

    if ((NULL == pids) || (NULL == infos)) {
6590
      taos_close(taos);
6591
      ERROR_EXIT("memory allocation failed for create threads\n");
6592
    }
6593

6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614
    for (uint64_t i = 0; i < nSqlCount; i++) {
        for (int j = 0; j < nConcurrent; j++) {
            uint64_t seq = i * nConcurrent + j;
            threadInfo *t_info = infos + seq;
            t_info->threadID = seq;
            t_info->querySeq = i;

            if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {

                char sqlStr[MAX_TB_NAME_SIZE*2];
                sprintf(sqlStr, "use %s", g_queryInfo.dbName);
                verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
                if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) {
                    taos_close(taos);
                    free(infos);
                    free(pids);
                    errorPrint( "use database %s failed!\n\n",
                        g_queryInfo.dbName);
                    return -1;
                }
            }
6615

6616
            t_info->taos = NULL;// TODO: workaround to use separate taos connection;
6617

6618 6619
            pthread_create(pids + seq, NULL, specifiedTableQuery,
                t_info);
6620
        }
6621
    }
6622
  } else {
6623
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
6624
  }
6625

6626 6627
  taos_close(taos);

6628 6629 6630
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
6631 6632 6633 6634
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
    pidsOfSub  = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t));
    infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo));
6635 6636

    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
6637 6638
      free(infos);
      free(pids);
6639

6640
      ERROR_EXIT("memory allocation failed for create threads\n");
6641
    }
6642

S
Shuduo Sang 已提交
6643
    int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
6644
    int threads = g_queryInfo.superQueryInfo.threadCnt;
6645

S
Shuduo Sang 已提交
6646
    int64_t a = ntables / threads;
6647 6648 6649 6650
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
6651

S
Shuduo Sang 已提交
6652
    int64_t b = 0;
6653 6654 6655
    if (threads != 0) {
      b = ntables % threads;
    }
6656

6657
    uint64_t startFrom = 0;
6658
    for (int i = 0; i < threads; i++) {
6659 6660
      threadInfo *t_info = infosOfSub + i;
      t_info->threadID = i;
6661

6662 6663 6664 6665
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
6666
      t_info->taos = NULL; // TODO: workaround to use separate taos connection;
6667
      pthread_create(pidsOfSub + i, NULL, superTableQuery, t_info);
6668 6669
    }

6670
    g_queryInfo.superQueryInfo.threadCnt = threads;
6671
  } else {
6672
    g_queryInfo.superQueryInfo.threadCnt = 0;
6673
  }
6674

6675 6676 6677 6678 6679 6680
  if ((nSqlCount > 0) && (nConcurrent > 0)) {
    for (int i = 0; i < nConcurrent; i++) {
      for (int j = 0; j < nSqlCount; j++) {
        pthread_join(pids[i * nSqlCount + j], NULL);
      }
    }
6681
  }
S
Shuaiqiang Chang 已提交
6682

6683
  tmfree((char*)pids);
6684
  tmfree((char*)infos);
6685

6686
  for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
6687
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
6688
  }
H
Hui Li 已提交
6689

6690
  tmfree((char*)pidsOfSub);
6691
  tmfree((char*)infosOfSub);
6692

6693
//  taos_close(taos);// TODO: workaround to use separate taos connection;
6694
  uint64_t endTs = taosGetTimestampMs();
6695

6696
  uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried +
6697 6698
    g_queryInfo.superQueryInfo.totalQueried;

6699
  fprintf(stderr, "==== completed total queries: %"PRIu64", the QPS of all threads: %10.3f====\n",
6700
          totalQueried,
6701
          (double)(totalQueried/((endTs-startTs)/1000.0)));
6702 6703 6704
  return 0;
}

6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719
static void stable_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
  if (res == NULL || taos_errno(res) != 0) {
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
    return;
  }

  if (param)
    appendResultToFile(res, ((threadInfo *)param)->fp);
  // tao_unscribe() will free result.
}

static void specified_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
6720
  if (res == NULL || taos_errno(res) != 0) {
6721 6722
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
6723 6724
    return;
  }
6725

6726
  if (param)
6727
    appendResultToFile(res, ((threadInfo *)param)->fp);
6728
  // tao_unscribe() will free result.
H
hzcheng 已提交
6729 6730
}

6731
static TAOS_SUB* subscribeImpl(
6732 6733 6734 6735
        QUERY_CLASS class,
        threadInfo *pThreadInfo,
        char *sql, char* topic, bool restart, uint64_t interval)
{
6736
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
6737

6738 6739 6740 6741
  if ((SPECIFIED_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) {
    tsub = taos_subscribe(
            pThreadInfo->taos,
6742
            restart,
6743
            topic, sql, specified_sub_callback, (void*)pThreadInfo,
6744
            g_queryInfo.specifiedQueryInfo.subscribeInterval);
6745 6746 6747 6748 6749 6750 6751
  } else if ((STABLE_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) {
    tsub = taos_subscribe(
            pThreadInfo->taos,
            restart,
            topic, sql, stable_sub_callback, (void*)pThreadInfo,
            g_queryInfo.superQueryInfo.subscribeInterval);
6752
  } else {
6753 6754
    tsub = taos_subscribe(
            pThreadInfo->taos,
6755
            restart,
6756
            topic, sql, NULL, NULL, interval);
6757
  }
6758

6759 6760 6761
  if (tsub == NULL) {
    printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
    return NULL;
6762
  }
6763

6764 6765
  return tsub;
}
H
hzcheng 已提交
6766

6767 6768
static void *superSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6769
  char subSqlstr[MAX_QUERY_SQL_LENGTH];
6770
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT] = {0};
H
hzcheng 已提交
6771

6772 6773
  if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
      errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n",
6774 6775 6776 6777 6778
              pThreadInfo->ntables,
              MAX_QUERY_SQL_COUNT);
      exit(-1);
  }

6779
  if (pThreadInfo->taos == NULL) {
6780 6781 6782 6783 6784 6785 6786 6787
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6788
            pThreadInfo->threadID, taos_errstr(NULL));
6789 6790
      return NULL;
    } else {
6791
      pThreadInfo->taos = taos;
6792 6793 6794
    }
  }

6795 6796
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6797 6798
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6799 6800
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
6801 6802
    return NULL;
  }
6803

S
Shengliang Guan 已提交
6804 6805 6806
  char topic[32] = {0};
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831
      verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n",
              __func__, __LINE__,
              pThreadInfo->threadID,
              pThreadInfo->start_table_from,
              pThreadInfo->end_table_to, i);
      sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"",
              i, pThreadInfo->querySeq);
      memset(subSqlstr, 0, sizeof(subSqlstr));
      replaceChildTblName(
              g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq],
              subSqlstr, i);
      if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
        sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
      }

      debugPrint("%s() LN%d, [%d] subSqlstr: %s\n",
              __func__, __LINE__, pThreadInfo->threadID, subSqlstr);
      tsub[i] = subscribeImpl(
              STABLE_CLASS,
              pThreadInfo, subSqlstr, topic,
              g_queryInfo.superQueryInfo.subscribeRestart,
              g_queryInfo.superQueryInfo.subscribeInterval);
      if (NULL == tsub[i]) {
6832
        taos_close(pThreadInfo->taos);
6833 6834
        return NULL;
      }
S
Shengliang Guan 已提交
6835
  }
6836 6837

  // start loop to consume result
6838 6839 6840 6841
  int consumed[MAX_QUERY_SQL_COUNT];
  for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) {
    consumed[i] = 0;
  }
6842
  TAOS_RES* res = NULL;
6843

6844
  while(1) {
6845 6846
    for (uint64_t i = pThreadInfo->start_table_from;
            i <= pThreadInfo->end_table_to; i++) {
S
Shengliang Guan 已提交
6847 6848 6849
        if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
            continue;
        }
6850

6851
        res = taos_consume(tsub[i]);
S
Shengliang Guan 已提交
6852
        if (res) {
6853 6854
            if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
                sprintf(pThreadInfo->fp, "%s-%d",
6855 6856 6857 6858 6859 6860 6861 6862
                        g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                        pThreadInfo->threadID);
                appendResultToFile(res, pThreadInfo->fp);
            }
            if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
                sprintf(pThreadInfo->fp, "%s-%d",
                        g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                        pThreadInfo->threadID);
6863
                appendResultToFile(res, pThreadInfo->fp);
S
Shengliang Guan 已提交
6864
            }
6865
            consumed[i] ++;
6866 6867

            if ((g_queryInfo.superQueryInfo.subscribeKeepProgress)
6868 6869 6870 6871 6872 6873
                && (consumed[i] >=
                    g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
                printf("keepProgress:%d, resub super table query: %"PRIu64"\n",
                    g_queryInfo.superQueryInfo.subscribeKeepProgress,
                    pThreadInfo->querySeq);
                taos_unsubscribe(tsub,
6874
                    g_queryInfo.superQueryInfo.subscribeKeepProgress);
6875 6876 6877 6878 6879 6880 6881 6882
                consumed[i]= 0;
                tsub[i] = subscribeImpl(
                        STABLE_CLASS,
                        pThreadInfo, subSqlstr, topic,
                        g_queryInfo.superQueryInfo.subscribeRestart,
                        g_queryInfo.superQueryInfo.subscribeInterval
                    );
                if (NULL == tsub[i]) {
6883 6884 6885 6886
                    taos_close(pThreadInfo->taos);
                    return NULL;
                }
            }
6887
        }
H
hzcheng 已提交
6888 6889
    }
  }
6890
  taos_free_result(res);
6891

S
Shengliang Guan 已提交
6892 6893
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6894
    taos_unsubscribe(tsub[i], 0);
6895
  }
6896

6897
  taos_close(pThreadInfo->taos);
H
hzcheng 已提交
6898 6899 6900
  return NULL;
}

6901 6902
static void *specifiedSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6903
  TAOS_SUB*  tsub = NULL;
6904

6905
  if (pThreadInfo->taos == NULL) {
6906 6907 6908 6909 6910 6911 6912 6913
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6914
            pThreadInfo->threadID, taos_errstr(NULL));
6915 6916
      return NULL;
    } else {
6917
      pThreadInfo->taos = taos;
6918 6919 6920
    }
  }

6921 6922
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6923
  debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
6924 6925
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6926 6927
    return NULL;
  }
6928

S
Shengliang Guan 已提交
6929
  char topic[32] = {0};
6930 6931 6932 6933 6934 6935 6936 6937 6938 6939
  sprintf(topic, "taosdemo-subscribe-%"PRIu64"", pThreadInfo->querySeq);
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
      sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }
  tsub = subscribeImpl(
                      SPECIFIED_CLASS, pThreadInfo,
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
          topic,
6940
          g_queryInfo.specifiedQueryInfo.subscribeRestart,
6941 6942 6943 6944
          g_queryInfo.specifiedQueryInfo.subscribeInterval);
  if (NULL == tsub) {
      taos_close(pThreadInfo->taos);
      return NULL;
S
Shengliang Guan 已提交
6945
  }
6946

6947 6948
  // start loop to consume result
  TAOS_RES* res = NULL;
6949

6950
  int consumed;
6951

6952
  while(1) {
6953
      if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) {
6954 6955
        continue;
      }
6956

6957
      res = taos_consume(tsub);
6958
      if (res) {
6959 6960 6961 6962 6963 6964 6965
          if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
              sprintf(pThreadInfo->fp, "%s-%d",
                      g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
              appendResultToFile(res, pThreadInfo->fp);
          }

6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983
          consumed ++;
          if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress)
                && (consumed >=
                    g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
              printf("keepProgress:%d, resub specified query: %"PRIu64"\n",
                    g_queryInfo.specifiedQueryInfo.subscribeKeepProgress,
                    pThreadInfo->querySeq);
              consumed = 0;
              taos_unsubscribe(tsub,
                      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
              tsub = subscribeImpl(
                      SPECIFIED_CLASS,
                      pThreadInfo,
                      g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
                      topic,
                      g_queryInfo.specifiedQueryInfo.subscribeRestart,
                      g_queryInfo.specifiedQueryInfo.subscribeInterval);
              if (NULL == tsub) {
6984 6985
                taos_close(pThreadInfo->taos);
                return NULL;
6986 6987
              }
          }
6988 6989 6990
      }
  }
  taos_free_result(res);
6991
  taos_unsubscribe(tsub, 0);
6992
  taos_close(pThreadInfo->taos);
6993

H
hzcheng 已提交
6994 6995 6996
  return NULL;
}

6997
static int subscribeTestProcess() {
6998
  setupForAnsiEscape();
6999
  printfQueryMeta();
7000
  resetAfterAnsiEscape();
7001

7002
  prompt();
7003

7004 7005 7006 7007 7008
  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
7009
          g_queryInfo.port);
7010
  if (taos == NULL) {
7011 7012
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
7013 7014 7015
    exit(-1);
  }

7016
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
7017 7018
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
7019 7020 7021
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
7022 7023
  }

7024 7025
  taos_close(taos); // TODO: workaround to use separate taos connection;

7026 7027
  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
7028 7029 7030 7031 7032

  pthread_t  *pidsOfStable  = NULL;
  threadInfo *infosOfStable = NULL;

  //==== create threads for query for specified table
S
Shengliang Guan 已提交
7033
  if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) {
7034
    debugPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n",
7035
              __func__, __LINE__,
S
Shengliang Guan 已提交
7036 7037 7038 7039 7040 7041 7042 7043
              g_queryInfo.specifiedQueryInfo.sqlCount);
  } else {
    if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) {
        errorPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n",
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount);
        exit(-1);
    }
7044

7045 7046 7047 7048 7049 7050 7051 7052
    pids  = malloc(
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(pthread_t));
    infos = malloc(
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(threadInfo));
S
Shengliang Guan 已提交
7053 7054 7055 7056
    if ((NULL == pids) || (NULL == infos)) {
        errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
        exit(-1);
    }
7057

7058 7059 7060 7061 7062 7063 7064 7065 7066
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
            uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
            threadInfo *t_info = infos + seq;
            t_info->threadID = seq;
            t_info->querySeq = i;
            t_info->taos = NULL;  // TODO: workaround to use separate taos connection;
            pthread_create(pids + seq, NULL, specifiedSubscribe, t_info);
        }
S
Shengliang Guan 已提交
7067
    }
H
hzcheng 已提交
7068
  }
7069

7070 7071 7072 7073 7074 7075 7076
  //==== create threads for super table query
  if (g_queryInfo.superQueryInfo.sqlCount <= 0) {
    printf("%s() LN%d, super table query sqlCount %"PRIu64".\n",
              __func__, __LINE__,
              g_queryInfo.superQueryInfo.sqlCount);
  } else {
    if ((g_queryInfo.superQueryInfo.sqlCount > 0)
7077
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
7078 7079 7080
        pidsOfStable  = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
7081
            sizeof(pthread_t));
7082 7083 7084
        infosOfStable = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
7085
            sizeof(threadInfo));
7086 7087
        if ((NULL == pidsOfStable) || (NULL == infosOfStable)) {
            errorPrint("%s() LN%d, malloc failed for create threads\n",
7088
              __func__, __LINE__);
7089 7090 7091
            // taos_close(taos);
            exit(-1);
        }
7092

7093 7094
        int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
        int threads = g_queryInfo.superQueryInfo.threadCnt;
7095

7096 7097 7098 7099 7100
        int64_t a = ntables / threads;
        if (a < 1) {
            threads = ntables;
            a = 1;
        }
7101

7102 7103 7104 7105
        int64_t b = 0;
        if (threads != 0) {
            b = ntables % threads;
        }
7106

7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123
        for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            uint64_t startFrom = 0;
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                threadInfo *t_info = infosOfStable + seq;
                t_info->threadID = seq;
                t_info->querySeq = i;

                t_info->start_table_from = startFrom;
                t_info->ntables = j<b?a+1:a;
                t_info->end_table_to = j<b?startFrom+a:startFrom+a-1;
                startFrom = t_info->end_table_to + 1;
                t_info->taos = NULL; // TODO: workaround to use separate taos connection;
                pthread_create(pidsOfStable + seq,
                        NULL, superSubscribe, t_info);
            }
        }
7124

7125
        g_queryInfo.superQueryInfo.threadCnt = threads;
7126

7127 7128 7129 7130 7131 7132
        for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                pthread_join(pidsOfStable[seq], NULL);
            }
        }
7133
    }
H
hzcheng 已提交
7134
  }
7135

7136 7137 7138 7139 7140
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
    for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
        uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
        pthread_join(pids[seq], NULL);
    }
7141
  }
H
hzcheng 已提交
7142

7143
  tmfree((char*)pids);
7144
  tmfree((char*)infos);
H
hzcheng 已提交
7145

7146 7147
  tmfree((char*)pidsOfStable);
  tmfree((char*)infosOfStable);
7148
//   taos_close(taos);
7149
  return 0;
H
hzcheng 已提交
7150 7151
}

7152
static void initOfInsertMeta() {
7153
  memset(&g_Dbs, 0, sizeof(SDbs));
7154

7155
  // set default values
7156
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7157
  g_Dbs.port = 6030;
7158 7159
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
7160
  g_Dbs.threadCount = 2;
7161 7162

  g_Dbs.use_metric = g_args.use_metric;
L
Liu Tao 已提交
7163 7164
}

7165
static void initOfQueryMeta() {
7166
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
7167

7168
  // set default values
7169
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7170
  g_queryInfo.port = 6030;
7171 7172
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
L
Liu Tao 已提交
7173 7174
}

7175
static void setParaFromArg(){
7176
  if (g_args.host) {
7177
    tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE);
7178
  } else {
7179
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7180
  }
L
Liu Tao 已提交
7181

7182
  if (g_args.user) {
7183
    tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE);
7184
  }
7185 7186

  if (g_args.password) {
7187
    tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE);
7188 7189
  }

7190 7191
  if (g_args.port) {
    g_Dbs.port = g_args.port;
7192
  }
L
Liu Tao 已提交
7193

7194 7195 7196
  g_Dbs.threadCount = g_args.num_of_threads;
  g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;

7197 7198
  g_Dbs.dbCount = 1;
  g_Dbs.db[0].drop = 1;
7199

7200
  tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE);
7201
  g_Dbs.db[0].dbCfg.replica = g_args.replica;
7202
  tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
7203

7204
  tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
7205 7206 7207 7208 7209

  g_Dbs.use_metric = g_args.use_metric;
  g_Dbs.insert_only = g_args.insert_only;

  g_Dbs.do_aggreFunc = true;
L
Liu Tao 已提交
7210

7211 7212
  char dataString[STRING_LEN];
  char **data_type = g_args.datatype;
7213

7214
  memset(dataString, 0, STRING_LEN);
L
Liu Tao 已提交
7215

7216 7217
  if (strcasecmp(data_type[0], "BINARY") == 0
          || strcasecmp(data_type[0], "BOOL") == 0
7218
          || strcasecmp(data_type[0], "NCHAR") == 0 ) {
7219
    g_Dbs.do_aggreFunc = false;
L
Liu Tao 已提交
7220
  }
H
hzcheng 已提交
7221

7222 7223 7224 7225 7226
  if (g_args.use_metric) {
    g_Dbs.db[0].superTblCount = 1;
    tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE);
    g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
    g_Dbs.threadCount = g_args.num_of_threads;
7227
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
7228
    g_Dbs.asyncMode = g_args.async_mode;
7229

7230 7231 7232 7233
    g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
    g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
    g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
7234
    tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix,
7235 7236
            g_args.tb_prefix, MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
S
Shuduo Sang 已提交
7237
    g_Dbs.db[0].superTbls[0].insertMode = g_args.iface;
7238
    tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp,
7239
            "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
7240
    g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
7241

7242
    g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
7243
    g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len;
7244

7245 7246 7247 7248 7249
    g_Dbs.db[0].superTbls[0].columnCount = 0;
    for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
      if (data_type[i] == NULL) {
        break;
      }
7250

7251
      tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
7252
              data_type[i], MAX_TB_NAME_SIZE);
7253
      g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;
7254 7255
      g_Dbs.db[0].superTbls[0].columnCount++;
    }
7256

7257 7258 7259
    if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
      g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
    } else {
7260 7261 7262 7263
      for (int i = g_Dbs.db[0].superTbls[0].columnCount;
              i < g_args.num_of_CPR; i++) {
        tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
                "INT", MAX_TB_NAME_SIZE);
7264
        g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;
7265 7266 7267
        g_Dbs.db[0].superTbls[0].columnCount++;
      }
    }
7268

7269 7270
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType,
            "INT", MAX_TB_NAME_SIZE);
7271
    g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;
7272

7273 7274
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType,
            "BINARY", MAX_TB_NAME_SIZE);
7275 7276
    g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;
    g_Dbs.db[0].superTbls[0].tagCount = 2;
7277
  } else {
7278 7279
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
    g_Dbs.db[0].superTbls[0].tagCount = 0;
H
hzcheng 已提交
7280
  }
7281 7282 7283 7284 7285 7286
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
7287

7288 7289 7290
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
7291 7292 7293
    exit(-1);
  }

7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316 7317
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

7318
static void querySqlFile(TAOS* taos, char* sqlFile)
7319 7320 7321 7322 7323 7324
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
7325

7326 7327 7328 7329 7330 7331
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

7332
  double t = taosGetTimestampMs();
7333

7334
  while((read_len = tgetline(&line, &line_len, fp)) != -1) {
7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
7347
    }
7348 7349

    memcpy(cmd + cmd_len, line, read_len);
7350
    verbosePrint("%s() LN%d cmd: %s\n", __func__, __LINE__, cmd);
7351 7352 7353
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) {
        errorPrint("%s() LN%d, queryDbExec %s failed!\n",
               __func__, __LINE__, cmd);
7354 7355 7356 7357 7358
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
7359 7360
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
7361 7362
  }

7363
  t = taosGetTimestampMs() - t;
7364
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
7365

7366 7367 7368 7369
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
7370 7371
}

7372
static void testMetaFile() {
7373
    if (INSERT_TEST == g_args.test_mode) {
7374 7375
      if (g_Dbs.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
7376

7377
      insertTestProcess();
7378

7379
    } else if (QUERY_TEST == g_args.test_mode) {
7380 7381
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7382

7383
      queryTestProcess();
7384

7385
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
7386 7387
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7388

7389
      subscribeTestProcess();
7390

7391 7392 7393
    }  else {
      ;
    }
7394
}
7395

7396
static void queryResult() {
7397
  // query data
7398

7399 7400 7401 7402 7403
  pthread_t read_id;
  threadInfo *rInfo = malloc(sizeof(threadInfo));
  assert(rInfo);
  rInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
  rInfo->start_table_from = 0;
7404

7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439 7440
  //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
  if (g_args.use_metric) {
    rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
    rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
    rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
    tstrncpy(rInfo->tb_prefix,
          g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE);
  } else {
    rInfo->ntables = g_args.num_of_tables;
    rInfo->end_table_to = g_args.num_of_tables -1;
    tstrncpy(rInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE);
  }

  rInfo->taos = taos_connect(
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
          g_Dbs.port);
  if (rInfo->taos == NULL) {
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
    free(rInfo);
    exit(-1);
  }

  tstrncpy(rInfo->fp, g_Dbs.resultFile, MAX_FILE_NAME_LEN);

  if (!g_Dbs.use_metric) {
    pthread_create(&read_id, NULL, readTable, rInfo);
  } else {
    pthread_create(&read_id, NULL, readMetric, rInfo);
  }
  pthread_join(read_id, NULL);
  taos_close(rInfo->taos);
  free(rInfo);
7441 7442
}

7443 7444
static void testCmdLine() {

7445 7446 7447 7448 7449 7450 7451 7452 7453 7454
  if (strlen(configDir)) {
    wordexp_t full_path;
    if (wordexp(configDir, &full_path, 0) != 0) {
      errorPrint( "Invalid path %s\n", configDir);
      return;
    }
    taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
    wordfree(&full_path);
  }

7455 7456
  g_args.test_mode = INSERT_TEST;
  insertTestProcess();
7457

7458
  if (false == g_Dbs.insert_only)
7459
    queryResult();
7460 7461
}

7462 7463 7464
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

7465
  debugPrint("meta file: %s\n", g_args.metaFile);
7466 7467 7468

  if (g_args.metaFile) {
    initOfInsertMeta();
7469
    initOfQueryMeta();
7470 7471 7472 7473 7474 7475 7476 7477 7478 7479 7480 7481 7482

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
7483 7484 7485 7486
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
7487
          g_Dbs.port);
7488
      querySqlFile(qtaos, g_args.sqlFile);
7489 7490 7491 7492 7493
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
7494 7495 7496

    if (g_dupstr)
        free(g_dupstr);
H
hzcheng 已提交
7497
  }
7498 7499

  return 0;
H
hzcheng 已提交
7500
}
7501