taosdemo.c 248.0 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

21
#include <stdint.h>
S
slguan 已提交
22
#define _GNU_SOURCE
23 24
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
41
#else
S
TD-1057  
Shengliang Guan 已提交
42 43
  #include <regex.h>
  #include <stdio.h>
44
#endif
H
hzcheng 已提交
45

46
#include <assert.h>
47
#include <stdlib.h>
48 49
#include "cJSON.h"

50
#include "os.h"
H
hzcheng 已提交
51
#include "taos.h"
52
#include "taoserror.h"
H
Hui Li 已提交
53
#include "tutil.h"
S
slguan 已提交
54

55 56 57
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
58 59
extern char configDir[];

60 61 62 63
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

64 65
#define STR_INSERT_INTO     "INSERT INTO "

66 67 68 69 70 71
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};
72

73 74
#define MAX_RECORDS_PER_REQ     32766

75 76
#define HEAD_BUFF_LEN    1024*24  // 16*1024 + (192+32)*2 + insert into ..

77 78
#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
79
#define COND_BUF_LEN        BUFFER_SIZE - 30
80 81
#define MAX_USERNAME_SIZE  64
#define MAX_PASSWORD_SIZE  64
82
#define MAX_DB_NAME_SIZE   64
83
#define MAX_HOSTNAME_SIZE  64
84
#define MAX_TB_NAME_SIZE   64
85
#define MAX_DATA_SIZE      (16*1024)+20     // max record len: 16*1024, timestamp string and ,('') need extra space
86 87 88 89 90 91 92 93 94 95
#define MAX_NUM_DATATYPE   10
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
#define MAX_FILE_NAME_LEN  256

#define   MAX_SAMPLES_ONCE_FROM_FILE   10000
#define   MAX_NUM_DATATYPE 10

#define   MAX_DB_COUNT           8
96
#define   MAX_SUPER_TABLE_COUNT  200
97 98 99
#define   MAX_COLUMN_COUNT       1024
#define   MAX_TAG_COUNT          128

100
#define   MAX_QUERY_SQL_COUNT    100
101
#define   MAX_QUERY_SQL_LENGTH   1024
102 103

#define   MAX_DATABASE_COUNT     256
104
#define INPUT_BUF_LEN   256
105

106
#define DEFAULT_TIMESTAMP_STEP  1
107

108

109 110 111 112 113
typedef enum CREATE_SUB_TALBE_MOD_EN {
  PRE_CREATE_SUBTBL,
  AUTO_CREATE_SUBTBL,
  NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
114

115 116
typedef enum TALBE_EXISTS_EN {
  TBL_NO_EXISTS,
117
  TBL_ALREADY_EXISTS,
118 119 120
  TBL_EXISTS_BUTT
} TALBE_EXISTS_EN;

121
enum enumSYNC_MODE {
122 123
  SYNC_MODE,
  ASYNC_MODE,
124 125
  MODE_BUT
};
126

S
Shuduo Sang 已提交
127
enum enum_TAOS_INTERFACE {
128 129 130
    TAOSC_IFACE,
    REST_IFACE,
    STMT_IFACE,
131 132 133
    INTERFACE_BUT
};

134 135 136 137 138 139
typedef enum enumQUERY_CLASS {
    SPECIFIED_CLASS,
    STABLE_CLASS,
    CLASS_BUT
} QUERY_CLASS;

S
Shuduo Sang 已提交
140
typedef enum enum_PROGRESSIVE_OR_INTERLACE {
141 142 143
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
144
} PROG_OR_INTERLACE_MODE;
145

146
typedef enum enumQUERY_TYPE {
147
  NO_INSERT_TYPE,
148
  INSERT_TYPE,
149
  QUERY_TYPE_BUT
150
} QUERY_TYPE;
151 152 153 154 155 156 157

enum _show_db_index {
  TSDB_SHOW_DB_NAME_INDEX,
  TSDB_SHOW_DB_CREATED_TIME_INDEX,
  TSDB_SHOW_DB_NTABLES_INDEX,
  TSDB_SHOW_DB_VGROUPS_INDEX,
  TSDB_SHOW_DB_REPLICA_INDEX,
158
  TSDB_SHOW_DB_QUORUM_INDEX,
159 160 161 162 163 164 165 166 167 168
  TSDB_SHOW_DB_DAYS_INDEX,
  TSDB_SHOW_DB_KEEP_INDEX,
  TSDB_SHOW_DB_CACHE_INDEX,
  TSDB_SHOW_DB_BLOCKS_INDEX,
  TSDB_SHOW_DB_MINROWS_INDEX,
  TSDB_SHOW_DB_MAXROWS_INDEX,
  TSDB_SHOW_DB_WALLEVEL_INDEX,
  TSDB_SHOW_DB_FSYNC_INDEX,
  TSDB_SHOW_DB_COMP_INDEX,
  TSDB_SHOW_DB_CACHELAST_INDEX,
169
  TSDB_SHOW_DB_PRECISION_INDEX,
170 171 172 173 174 175 176 177 178 179
  TSDB_SHOW_DB_UPDATE_INDEX,
  TSDB_SHOW_DB_STATUS_INDEX,
  TSDB_MAX_SHOW_DB
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
  TSDB_SHOW_STABLES_NAME_INDEX,
  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
  TSDB_SHOW_STABLES_COLUMNS_INDEX,
180 181
  TSDB_SHOW_STABLES_METRIC_INDEX,
  TSDB_SHOW_STABLES_UID_INDEX,
182
  TSDB_SHOW_STABLES_TID_INDEX,
183
  TSDB_SHOW_STABLES_VGID_INDEX,
184
  TSDB_MAX_SHOW_STABLES
185 186
};

187 188 189 190 191 192 193 194 195 196 197 198 199 200
enum _describe_table_index {
  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
  TSDB_MAX_DESCRIBE_METRIC
};

typedef struct {
  char field[TSDB_COL_NAME_LEN + 1];
  char type[16];
  int length;
  char note[128];
} SColDes;
H
hzcheng 已提交
201 202

/* Used by main to communicate with parse_opt. */
203 204
static char *g_dupstr = NULL;

205 206
typedef struct SArguments_S {
  char *   metaFile;
207
  uint32_t test_mode;
208 209
  char *   host;
  uint16_t port;
S
Shuduo Sang 已提交
210
  uint16_t iface;
211 212 213 214 215 216 217
  char *   user;
  char *   password;
  char *   database;
  int      replica;
  char *   tb_prefix;
  char *   sqlFile;
  bool     use_metric;
218
  bool     drop_database;
219
  bool     insert_only;
220
  bool     answer_yes;
221
  bool     debug_print;
222
  bool     verbose_print;
223
  bool     performance_print;
224
  char *   output_file;
225
  bool     async_mode;
226
  char *   datatype[MAX_NUM_DATATYPE + 1];
227 228 229 230
  uint32_t len_of_binary;
  uint32_t num_of_CPR;
  uint32_t num_of_threads;
  uint64_t insert_interval;
231
  int64_t  query_times;
232 233 234
  uint64_t interlace_rows;
  uint64_t num_of_RPR;                  // num_of_records_per_req
  uint64_t max_sql_len;
235 236
  int64_t  num_of_tables;
  int64_t  num_of_DPT;
237
  int      abort;
238 239
  int      disorderRatio;               // 0: no disorder, >0: x%
  int      disorderRange;               // ms or us by database precision
240
  uint32_t method_of_delete;
241
  char **  arg_list;
242 243
  uint64_t totalInsertRows;
  uint64_t totalAffectedRows;
244 245 246
} SArguments;

typedef struct SColumn_S {
247 248 249
  char      field[TSDB_COL_NAME_LEN + 1];
  char      dataType[MAX_TB_NAME_SIZE];
  uint32_t  dataLen;
250
  char      note[128];
251 252 253
} StrColumn;

typedef struct SSuperTable_S {
254
  char         sTblName[MAX_TB_NAME_SIZE+1];
255
  int64_t      childTblCount;
256
  bool         childTblExists;          // 0: no, 1: yes
257 258
  uint64_t     batchCreateTableNum;     // 0: no batch,  > 0: batch table number in one sql
  uint8_t      autoCreateTable;         // 0: create sub table, 1: auto create sub table
259
  char         childTblPrefix[MAX_TB_NAME_SIZE];
260
  char         dataSource[MAX_TB_NAME_SIZE+1];  // rand_gen or sample
261
  uint16_t     insertMode;              // 0: taosc, 1: rest, 2: stmt
262
  int64_t      childTblLimit;
263
  uint64_t     childTblOffset;
264

265
//  int          multiThreadWriteOneTbl;  // 0: no, 1: yes
266
  uint64_t     interlaceRows;           //
267 268
  int          disorderRatio;           // 0: no disorder, >0: x%
  int          disorderRange;           // ms or us by database precision
269
  uint64_t     maxSqlLen;               //
270

271
  uint64_t     insertInterval;          // insert interval, will override global insert interval
272
  int64_t      insertRows;
273
  int64_t      timeStampStep;
274
  char         startTimestamp[MAX_TB_NAME_SIZE];
275
  char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
276 277
  char         sampleFile[MAX_FILE_NAME_LEN+1];
  char         tagsFile[MAX_FILE_NAME_LEN+1];
278

279
  uint32_t     columnCount;
280
  StrColumn    columns[MAX_COLUMN_COUNT];
281
  uint32_t     tagCount;
282 283 284
  StrColumn    tags[MAX_TAG_COUNT];

  char*        childTblName;
285
  char*        colsOfCreateChildTable;
286 287
  uint64_t     lenOfOneRow;
  uint64_t     lenOfTagOfOneRow;
288 289 290 291 292

  char*        sampleDataBuf;
  //int          sampleRowCount;
  //int          sampleUsePos;

293
  uint32_t     tagSource;    // 0: rand, 1: tag sample
294
  char*        tagDataBuf;
295 296
  uint32_t     tagSampleCount;
  uint32_t     tagUsePos;
297

298
  // statistics
299 300
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
301 302 303 304 305
} SSuperTable;

typedef struct {
  char     name[TSDB_DB_NAME_LEN + 1];
  char     create_time[32];
S
Shuduo Sang 已提交
306
  int64_t  ntables;
307
  int32_t  vgroups;
308 309
  int16_t  replica;
  int16_t  quorum;
310
  int16_t  days;
311 312 313 314 315 316 317 318 319 320 321 322 323 324
  char     keeplist[32];
  int32_t  cache; //MB
  int32_t  blocks;
  int32_t  minrows;
  int32_t  maxrows;
  int8_t   wallevel;
  int32_t  fsync;
  int8_t   comp;
  int8_t   cachelast;
  char     precision[8];   // time resolution
  int8_t   update;
  char     status[16];
} SDbInfo;

325
typedef struct SDbCfg_S {
326
//  int       maxtablesPerVnode;
327 328
  uint32_t  minRows;        // 0 means default
  uint32_t  maxRows;        // 0 means default
329 330
  int       comp;
  int       walLevel;
331
  int       cacheLast;
332
  int       fsync;
333 334 335 336 337 338 339
  int       replica;
  int       update;
  int       keep;
  int       days;
  int       cache;
  int       blocks;
  int       quorum;
340
  char      precision[MAX_TB_NAME_SIZE];
341 342 343 344
} SDbCfg;

typedef struct SDataBase_S {
  char         dbName[MAX_DB_NAME_SIZE];
345
  bool         drop;  // 0: use exists, 1: if exists, drop then new create
346
  SDbCfg       dbCfg;
347
  uint64_t     superTblCount;
348 349 350 351
  SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
} SDataBase;

typedef struct SDbs_S {
352
  char         cfgDir[MAX_FILE_NAME_LEN+1];
353
  char         host[MAX_HOSTNAME_SIZE];
354 355
  struct sockaddr_in serv_addr;

356
  uint16_t     port;
357 358
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
359
  char         resultFile[MAX_FILE_NAME_LEN+1];
360 361 362
  bool         use_metric;
  bool         insert_only;
  bool         do_aggreFunc;
363
  bool         asyncMode;
364

365 366 367
  uint32_t     threadCount;
  uint32_t     threadCountByCreateTbl;
  uint32_t     dbCount;
368 369 370
  SDataBase    db[MAX_DB_COUNT];

  // statistics
371 372
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
373

374 375
} SDbs;

376
typedef struct SpecifiedQueryInfo_S {
377 378 379
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint64_t     concurrent;
  uint64_t     sqlCount;
380
  uint32_t     asyncMode; // 0: sync, 1: async
381 382
  uint64_t     subscribeInterval; // ms
  uint64_t     queryTimes;
383
  bool         subscribeRestart;
384
  int          subscribeKeepProgress;
385
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
386
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
387
  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
388
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
389
  uint64_t     totalQueried;
390
} SpecifiedQueryInfo;
391

392
typedef struct SuperQueryInfo_S {
393
  char         sTblName[MAX_TB_NAME_SIZE+1];
394 395
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint32_t     threadCnt;
396
  uint32_t     asyncMode; // 0: sync, 1: async
397
  uint64_t     subscribeInterval; // ms
398
  bool         subscribeRestart;
399
  int          subscribeKeepProgress;
400
  uint64_t     queryTimes;
401
  int64_t      childTblCount;
402
  char         childTblPrefix[MAX_TB_NAME_SIZE];
403
  uint64_t     sqlCount;
404
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
405
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
406
  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
407
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
408

409
  char*        childTblName;
410
  uint64_t     totalQueried;
411
} SuperQueryInfo;
412 413

typedef struct SQueryMetaInfo_S {
414
  char         cfgDir[MAX_FILE_NAME_LEN+1];
415
  char         host[MAX_HOSTNAME_SIZE];
416
  uint16_t     port;
417
  struct       sockaddr_in serv_addr;
418 419
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
420
  char         dbName[MAX_DB_NAME_SIZE+1];
421
  char         queryMode[MAX_TB_NAME_SIZE];  // taosc, rest
422

423
  SpecifiedQueryInfo  specifiedQueryInfo;
424
  SuperQueryInfo      superQueryInfo;
425
  uint64_t     totalQueried;
426 427 428
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
429
  TAOS *    taos;
430
  TAOS_STMT *stmt;
431 432 433 434 435 436 437
  int       threadID;
  char      db_name[MAX_DB_NAME_SIZE+1];
  uint32_t  time_precision;
  char      fp[4096];
  char      tb_prefix[MAX_TB_NAME_SIZE];
  uint64_t  start_table_from;
  uint64_t  end_table_to;
S
Shuduo Sang 已提交
438
  int64_t   ntables;
439 440 441 442
  uint64_t  data_of_rate;
  int64_t   start_time;
  char*     cols;
  bool      use_metric;
443
  SSuperTable* superTblInfo;
S
Shuduo Sang 已提交
444
  char      *buffer;    // sql cmd buffer
445 446

  // for async insert
447 448
  tsem_t    lock_sem;
  int64_t   counter;
449 450
  uint64_t  st;
  uint64_t  et;
451
  uint64_t  lastTs;
452

453
  // sample data
454
  int64_t   samplePos;
455
  // statistics
456 457
  uint64_t  totalInsertRows;
  uint64_t  totalAffectedRows;
458 459

  // insert delay statistics
460 461 462 463 464
  uint64_t  cntDelay;
  uint64_t  totalDelay;
  uint64_t  avgDelay;
  uint64_t  maxDelay;
  uint64_t  minDelay;
465

466
  // seq of query or subscribe
467
  uint64_t  querySeq;   // sequence number of sql command
468

469 470
} threadInfo;

471
#ifdef WINDOWS
472 473
#define _CRT_RAND_S

474
#include <windows.h>
475 476 477 478 479
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
480 481
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
482 483
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
484 485 486 487

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

488
static void setupForAnsiEscape(void) {
489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
507
  }
508 509
}

510
static void resetAfterAnsiEscape(void) {
511
  // Reset colors
512
  printf("\x1b[0m");
513 514 515 516 517 518

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
519

520
static int taosRandom()
521 522 523 524 525 526
{
    int number;
    rand_s(&number);

    return number;
}
527
#else   // Not windows
528
static void setupForAnsiEscape(void) {}
529

530
static void resetAfterAnsiEscape(void) {
531 532 533
  // Reset colors
  printf("\x1b[0m");
}
534

535 536
#include <time.h>

537
static int taosRandom()
538
{
539
  return rand();
540 541
}

542
#endif // ifdef Windows
543

544
static void prompt();
545
static int createDatabasesAndStables();
546
static void createChildTables();
547
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet);
548 549
static int postProceSql(char *host, struct sockaddr_in *pServAddr,
        uint16_t port, char* sqlstr, char *resultFile);
550 551 552 553 554 555 556

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
557
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)",
558 559
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

560
SArguments g_args = {
561 562 563 564
                     NULL,            // metaFile
                     0,               // test_mode
                     "127.0.0.1",     // host
                     6030,            // port
S
Shuduo Sang 已提交
565
                     TAOSC_IFACE,     // iface
566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597
                     "root",          // user
                     #ifdef _TD_POWER_
                     "powerdb",      // password
                     #else
                     "taosdata",      // password
                     #endif
                     "test",          // database
                     1,               // replica
                     "t",             // tb_prefix
                     NULL,            // sqlFile
                     true,            // use_metric
                     true,            // drop_database
                     true,            // insert_only
                     false,           // debug_print
                     false,           // verbose_print
                     false,           // performance statistic print
                     false,           // answer_yes;
                     "./output.txt",  // output_file
                     0,               // mode : sync or async
                     {
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     },
                     16,              // len_of_binary
                     4,               // num_of_CPR
                     10,              // num_of_connections/thread
                     0,               // insert_interval
                     1,               // query_times
                     0,               // interlace_rows;
                     30000,           // num_of_RPR
sangshuduo's avatar
sangshuduo 已提交
598
                     (1024*1024),         // max_sql_len
599 600 601 602 603 604 605
                     10000,           // num_of_tables
                     10000,           // num_of_DPT
                     0,               // abort
                     0,               // disorderRatio
                     1000,            // disorderRange
                     1,               // method_of_delete
                     NULL             // arg_list
606 607 608 609 610
};



static SDbs            g_Dbs;
611
static int64_t         g_totalChildTables = 0;
612 613 614 615
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

#define debugPrint(fmt, ...) \
616
    do { if (g_args.debug_print || g_args.verbose_print) \
617
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
618

619
#define verbosePrint(fmt, ...) \
620 621
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
622

623 624 625 626
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)

627 628 629 630
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)


631
///////////////////////////////////////////////////
H
hzcheng 已提交
632

633 634
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

635 636 637 638 639 640 641 642
#ifndef TAOSDEMO_COMMIT_SHA1
#define TAOSDEMO_COMMIT_SHA1 "unknown"
#endif

#ifndef TD_VERNUMBER
#define TD_VERNUMBER    "unknown"
#endif

643
#ifndef TAOSDEMO_STATUS
644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660
#define TAOSDEMO_STATUS "unknown"
#endif

static void printVersion() {
    char tdengine_ver[] = TD_VERNUMBER;
    char taosdemo_ver[] = TAOSDEMO_COMMIT_SHA1;
    char taosdemo_status[] = TAOSDEMO_STATUS;

    if (strlen(taosdemo_status) == 0) {
        printf("taosdemo verison %s-%s\n",
                tdengine_ver, taosdemo_ver);
    } else {
        printf("taosdemo verison %s-%s, status:%s\n",
                tdengine_ver, taosdemo_ver, taosdemo_status);
    }
}

661
static void printHelp() {
662
  char indent[10] = "        ";
663
  printf("%s%s%s%s\n", indent, "-f", indent,
664
          "The meta file to the execution procedure. Default is './meta.json'.");
665
  printf("%s%s%s%s\n", indent, "-u", indent,
666 667
          "The TDengine user name to use when connecting to the server. Default is 'root'.");
#ifdef _TD_POWER_
668
  printf("%s%s%s%s\n", indent, "-P", indent,
669
          "The password to use when connecting to the server. Default is 'powerdb'.");
670
  printf("%s%s%s%s\n", indent, "-c", indent,
671 672
          "Configuration directory. Default is '/etc/power/'.");
#else
673
  printf("%s%s%s%s\n", indent, "-P", indent,
674
          "The password to use when connecting to the server. Default is 'taosdata'.");
675
  printf("%s%s%s%s\n", indent, "-c", indent,
676
          "Configuration directory. Default is '/etc/taos/'.");
677
#endif
678
  printf("%s%s%s%s\n", indent, "-h", indent,
679
          "The host to connect to TDengine. Default is localhost.");
680
  printf("%s%s%s%s\n", indent, "-p", indent,
681
          "The TCP/IP port number to use for the connection. Default is 0.");
682 683
  printf("%s%s%s%s\n", indent, "-I", indent,
          "The interface (taosc, rest, and stmt) taosdemo uses. Default is 'taosc'.");
684
  printf("%s%s%s%s\n", indent, "-d", indent,
685
          "Destination database. Default is 'test'.");
686
  printf("%s%s%s%s\n", indent, "-a", indent,
687
          "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
688
  printf("%s%s%s%s\n", indent, "-m", indent,
689 690
          "Table prefix name. Default is 't'.");
  printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
691
  printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
692
  printf("%s%s%s%s\n", indent, "-o", indent,
693
          "Direct output to the named file. Default is './output.txt'.");
694
  printf("%s%s%s%s\n", indent, "-q", indent,
695
          "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC.");
696
  printf("%s%s%s%s\n", indent, "-b", indent,
sangshuduo's avatar
sangshuduo 已提交
697
          "The data_type of columns, default: INT,INT,INT,INT.");
698
  printf("%s%s%s%s\n", indent, "-w", indent,
699
          "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
700 701 702
  printf("%s%s%s%s%d\n", indent, "-l", indent,
          "The number of columns per record. Default is 4. Max values is ",
       MAX_NUM_DATATYPE);
703
  printf("%s%s%s%s\n", indent, "-T", indent,
704
          "The number of threads. Default is 10.");
705
  printf("%s%s%s%s\n", indent, "-i", indent,
706
          "The sleep time (ms) between insertion. Default is 0.");
707
  printf("%s%s%s%s\n", indent, "-r", indent,
708
          "The number of records per request. Default is 30000.");
709
  printf("%s%s%s%s\n", indent, "-t", indent,
710
          "The number of tables. Default is 10000.");
711
  printf("%s%s%s%s\n", indent, "-n", indent,
712 713 714
          "The number of records per table. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
  printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
715
  printf("%s%s%s%s\n", indent, "-O", indent,
716
          "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order.");
717
  printf("%s%s%s%s\n", indent, "-R", indent,
718
          "Out of order data's range, ms, default is 1000.");
719
  printf("%s%s%s%s\n", indent, "-g", indent,
720
          "Print debug info.");
721
  printf("%s%s%s\n", indent, "-V, --version\t",
722
          "Print version info.");
723 724
  printf("%s%s%s%s\n", indent, "--help\t", indent,
          "Print command line arguments list info.");
725
/*    printf("%s%s%s%s\n", indent, "-D", indent,
726 727 728 729
          "if elete database if exists. 0: no, 1: yes, default is 1");
          */
}

730 731 732 733 734 735 736 737 738 739 740 741 742 743 744
static bool isStringNumber(char *input)
{
  int len = strlen(input);
  if (0 == len) {
    return false;
  }

  for (int i = 0; i < len; i++) {
    if (!isdigit(input[i]))
      return false;
  }

  return true;
}

745
static void parse_args(int argc, char *argv[], SArguments *arguments) {
746 747 748 749 750

  for (int i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-f") == 0) {
      arguments->metaFile = argv[++i];
    } else if (strcmp(argv[i], "-c") == 0) {
H
Haojun Liao 已提交
751 752 753 754 755
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-c need a valid path following!\n");
        exit(EXIT_FAILURE);
      }
756
      tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN);
757
    } else if (strcmp(argv[i], "-h") == 0) {
H
Haojun Liao 已提交
758 759 760 761 762
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-h need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
763 764
      arguments->host = argv[++i];
    } else if (strcmp(argv[i], "-p") == 0) {
765
      if ((argc == i+1) ||
766
        (!isStringNumber(argv[i+1]))) {
767 768 769 770
        printHelp();
        errorPrint("%s", "\n\t-p need a number following!\n");
        exit(EXIT_FAILURE);
      }
771
      arguments->port = atoi(argv[++i]);
772 773 774 775 776 777 778 779
    } else if (strcmp(argv[i], "-I") == 0) {
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-I need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
      ++i;
      if (0 == strcasecmp(argv[i], "taosc")) {
S
Shuduo Sang 已提交
780
          arguments->iface = TAOSC_IFACE;
781
      } else if (0 == strcasecmp(argv[i], "rest")) {
S
Shuduo Sang 已提交
782
          arguments->iface = REST_IFACE;
783
      } else if (0 == strcasecmp(argv[i], "stmt")) {
S
Shuduo Sang 已提交
784
          arguments->iface = STMT_IFACE;
785 786 787 788
      } else {
        errorPrint("%s", "\n\t-I need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
789
    } else if (strcmp(argv[i], "-u") == 0) {
H
Haojun Liao 已提交
790 791 792 793 794
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-u need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
795 796
      arguments->user = argv[++i];
    } else if (strcmp(argv[i], "-P") == 0) {
H
Haojun Liao 已提交
797 798 799 800 801
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-P need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
802 803
      arguments->password = argv[++i];
    } else if (strcmp(argv[i], "-o") == 0) {
H
Haojun Liao 已提交
804 805 806 807 808
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-o need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
809 810
      arguments->output_file = argv[++i];
    } else if (strcmp(argv[i], "-s") == 0) {
H
Haojun Liao 已提交
811 812 813 814 815
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-s need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
816 817
      arguments->sqlFile = argv[++i];
    } else if (strcmp(argv[i], "-q") == 0) {
818 819
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
820
        printHelp();
821
        errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n");
822 823
        exit(EXIT_FAILURE);
      }
824
      arguments->async_mode = atoi(argv[++i]);
825
    } else if (strcmp(argv[i], "-T") == 0) {
826 827
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
828 829 830 831
        printHelp();
        errorPrint("%s", "\n\t-T need a number following!\n");
        exit(EXIT_FAILURE);
      }
832 833
      arguments->num_of_threads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
834 835
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
836 837 838 839
        printHelp();
        errorPrint("%s", "\n\t-i need a number following!\n");
        exit(EXIT_FAILURE);
      }
840
      arguments->insert_interval = atoi(argv[++i]);
841
    } else if (strcmp(argv[i], "-qt") == 0) {
842
      if ((argc == i+1)
843
              || (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
844
        printHelp();
845
        errorPrint("%s", "\n\t-qt need a number following!\n");
H
Haojun Liao 已提交
846 847
        exit(EXIT_FAILURE);
      }
848
      arguments->query_times = atoi(argv[++i]);
849
    } else if (strcmp(argv[i], "-B") == 0) {
850 851
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
852 853 854 855
        printHelp();
        errorPrint("%s", "\n\t-B need a number following!\n");
        exit(EXIT_FAILURE);
      }
856
      arguments->interlace_rows = atoi(argv[++i]);
857
    } else if (strcmp(argv[i], "-r") == 0) {
858 859
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
860 861 862 863
        printHelp();
        errorPrint("%s", "\n\t-r need a number following!\n");
        exit(EXIT_FAILURE);
      }
864 865
      arguments->num_of_RPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-t") == 0) {
866 867 868 869 870 871
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-t need a number following!\n");
        exit(EXIT_FAILURE);
      }
872 873
      arguments->num_of_tables = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
874 875 876 877 878 879
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-n need a number following!\n");
        exit(EXIT_FAILURE);
      }
880 881
      arguments->num_of_DPT = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-d") == 0) {
H
Haojun Liao 已提交
882 883 884 885 886
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-d need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
887 888
      arguments->database = argv[++i];
    } else if (strcmp(argv[i], "-l") == 0) {
889 890 891 892 893 894
      if (argc == i+1) {
        if (!isStringNumber(argv[i+1])) {
            printHelp();
            errorPrint("%s", "\n\t-l need a number following!\n");
            exit(EXIT_FAILURE);
        }
895
      }
896
      arguments->num_of_CPR = atoi(argv[++i]);
897 898 899 900 901 902 903 904 905 906 907

      if (arguments->num_of_CPR > MAX_NUM_DATATYPE) {
          printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE);
          prompt();
          arguments->num_of_CPR = MAX_NUM_DATATYPE;
      }

      for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) {
          arguments->datatype[col] = NULL;
      }

908
    } else if (strcmp(argv[i], "-b") == 0) {
909 910 911 912 913
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-b need valid string following!\n");
        exit(EXIT_FAILURE);
      }
914 915
      ++i;
      if (strstr(argv[i], ",") == NULL) {
916
        // only one col
917
        if (strcasecmp(argv[i], "INT")
918 919 920 921 922 923 924 925
                && strcasecmp(argv[i], "FLOAT")
                && strcasecmp(argv[i], "TINYINT")
                && strcasecmp(argv[i], "BOOL")
                && strcasecmp(argv[i], "SMALLINT")
                && strcasecmp(argv[i], "BIGINT")
                && strcasecmp(argv[i], "DOUBLE")
                && strcasecmp(argv[i], "BINARY")
                && strcasecmp(argv[i], "NCHAR")) {
926
          printHelp();
927
          errorPrint("%s", "-b: Invalid data_type!\n");
928
          exit(EXIT_FAILURE);
929
        }
930
        arguments->datatype[0] = argv[i];
931
      } else {
932
        // more than one col
933
        int index = 0;
934 935
        g_dupstr = strdup(argv[i]);
        char *running = g_dupstr;
936
        char *token = strsep(&running, ",");
937
        while(token != NULL) {
938
          if (strcasecmp(token, "INT")
939 940 941 942 943 944 945 946
                  && strcasecmp(token, "FLOAT")
                  && strcasecmp(token, "TINYINT")
                  && strcasecmp(token, "BOOL")
                  && strcasecmp(token, "SMALLINT")
                  && strcasecmp(token, "BIGINT")
                  && strcasecmp(token, "DOUBLE")
                  && strcasecmp(token, "BINARY")
                  && strcasecmp(token, "NCHAR")) {
947
            printHelp();
948
            free(g_dupstr);
949
            errorPrint("%s", "-b: Invalid data_type!\n");
950
            exit(EXIT_FAILURE);
H
hzcheng 已提交
951
          }
952
          arguments->datatype[index++] = token;
953 954
          token = strsep(&running, ",");
          if (index >= MAX_NUM_DATATYPE) break;
S
TD-1057  
Shengliang Guan 已提交
955
        }
956
        arguments->datatype[index] = NULL;
957
      }
958
    } else if (strcmp(argv[i], "-w") == 0) {
959
      if ((argc == i+1) ||
960
        (!isStringNumber(argv[i+1]))) {
961 962 963 964
        printHelp();
        errorPrint("%s", "\n\t-w need a number following!\n");
        exit(EXIT_FAILURE);
      }
965 966
      arguments->len_of_binary = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-m") == 0) {
H
Haojun Liao 已提交
967
      if ((argc == i+1) ||
968
        (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
969 970 971 972
        printHelp();
        errorPrint("%s", "\n\t-m need a number following!\n");
        exit(EXIT_FAILURE);
      }
973
      arguments->tb_prefix = argv[++i];
974 975
    } else if (strcmp(argv[i], "-N") == 0) {
      arguments->use_metric = false;
976
    } else if (strcmp(argv[i], "-x") == 0) {
977
      arguments->insert_only = false;
978
    } else if (strcmp(argv[i], "-y") == 0) {
979
      arguments->answer_yes = true;
980 981
    } else if (strcmp(argv[i], "-g") == 0) {
      arguments->debug_print = true;
982 983
    } else if (strcmp(argv[i], "-gg") == 0) {
      arguments->verbose_print = true;
984 985
    } else if (strcmp(argv[i], "-pp") == 0) {
      arguments->performance_print = true;
986
    } else if (strcmp(argv[i], "-O") == 0) {
987 988 989 990 991 992
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-O need a number following!\n");
        exit(EXIT_FAILURE);
      }
993

994
      arguments->disorderRatio = atoi(argv[++i]);
995

996
      if (arguments->disorderRatio > 50) {
997
        arguments->disorderRatio = 50;
998
      }
999

1000
      if (arguments->disorderRatio < 0) {
1001
        arguments->disorderRatio = 0;
1002
      }
1003

1004
    } else if (strcmp(argv[i], "-R") == 0) {
1005 1006 1007 1008 1009 1010
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-R need a number following!\n");
        exit(EXIT_FAILURE);
      }
1011

1012
      arguments->disorderRange = atoi(argv[++i]);
1013 1014 1015
      if (arguments->disorderRange < 0)
        arguments->disorderRange = 1000;

1016
    } else if (strcmp(argv[i], "-a") == 0) {
1017 1018 1019 1020 1021 1022
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-a need a number following!\n");
        exit(EXIT_FAILURE);
      }
1023 1024 1025
      arguments->replica = atoi(argv[++i]);
      if (arguments->replica > 3 || arguments->replica < 1) {
          arguments->replica = 1;
1026
      }
1027 1028
    } else if (strcmp(argv[i], "-D") == 0) {
      arguments->method_of_delete = atoi(argv[++i]);
1029 1030 1031
      if (arguments->method_of_delete > 3) {
        errorPrint("%s", "\n\t-D need a valud (0~3) number following!\n");
        exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1032
      }
1033 1034 1035 1036
    } else if ((strcmp(argv[i], "--version") == 0) ||
        (strcmp(argv[i], "-V") == 0)){
      printVersion();
      exit(0);
1037 1038 1039 1040 1041
    } else if (strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else {
      printHelp();
1042
      errorPrint("%s", "ERROR: wrong options\n");
1043
      exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1044 1045
    }
  }
1046

1047
  if (((arguments->debug_print) && (arguments->metaFile == NULL))
1048
          || arguments->verbose_print) {
1049
    printf("###################################################################\n");
1050
    printf("# meta file:                         %s\n", arguments->metaFile);
1051
    printf("# Server IP:                         %s:%hu\n",
1052 1053
            arguments->host == NULL ? "localhost" : arguments->host,
            arguments->port );
1054 1055
    printf("# User:                              %s\n", arguments->user);
    printf("# Password:                          %s\n", arguments->password);
1056 1057
    printf("# Use metric:                        %s\n",
            arguments->use_metric ? "true" : "false");
1058
    if (*(arguments->datatype)) {
1059 1060 1061 1062 1063 1064 1065
        printf("# Specified data type:               ");
        for (int i = 0; i < MAX_NUM_DATATYPE; i++)
            if (arguments->datatype[i])
               printf("%s,", arguments->datatype[i]);
            else
                break;
        printf("\n");
1066
    }
1067
    printf("# Insertion interval:                %"PRIu64"\n",
1068
            arguments->insert_interval);
1069
    printf("# Number of records per req:         %"PRIu64"\n",
1070
            arguments->num_of_RPR);
1071
    printf("# Max SQL length:                    %"PRIu64"\n",
1072
            arguments->max_sql_len);
1073
    printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
1074
    printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
1075
    printf("# Number of Tables:                  %"PRId64"\n",
1076
            arguments->num_of_tables);
1077
    printf("# Number of Data per Table:          %"PRId64"\n",
1078
            arguments->num_of_DPT);
1079 1080 1081 1082 1083
    printf("# Database name:                     %s\n", arguments->database);
    printf("# Table prefix:                      %s\n", arguments->tb_prefix);
    if (arguments->disorderRatio) {
      printf("# Data order:                        %d\n", arguments->disorderRatio);
      printf("# Data out of order rate:            %d\n", arguments->disorderRange);
1084

1085 1086 1087 1088
    }
    printf("# Delete method:                     %d\n", arguments->method_of_delete);
    printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
    printf("# Print debug info:                  %d\n", arguments->debug_print);
1089
    printf("# Print verbose info:                %d\n", arguments->verbose_print);
1090
    printf("###################################################################\n");
1091 1092

    prompt();
1093
  }
1094
}
H
hzcheng 已提交
1095

1096 1097
static bool getInfoFromJsonFile(char* file);
static void init_rand_data();
1098
static void tmfclose(FILE *fp) {
1099 1100 1101 1102
  if (NULL != fp) {
    fclose(fp);
  }
}
1103

1104
static void tmfree(char *buf) {
1105 1106
  if (NULL != buf) {
    free(buf);
H
Hui Li 已提交
1107
  }
1108
}
H
Hui Li 已提交
1109

1110
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
1111 1112 1113
  int i;
  TAOS_RES *res = NULL;
  int32_t   code = -1;
1114

S
Shuduo Sang 已提交
1115
  for (i = 0; i < 5 /* retry */; i++) {
1116 1117 1118
    if (NULL != res) {
      taos_free_result(res);
      res = NULL;
H
hzcheng 已提交
1119
    }
1120

1121 1122 1123 1124
    res = taos_query(taos, command);
    code = taos_errno(res);
    if (0 == code) {
      break;
1125
    }
H
hzcheng 已提交
1126 1127
  }

1128
  if (code != 0) {
1129 1130
    if (!quiet) {
      debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
1131
      errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res));
1132
    }
1133 1134 1135
    taos_free_result(res);
    //taos_close(taos);
    return -1;
1136
  }
H
hzcheng 已提交
1137

1138 1139 1140 1141
  if (INSERT_TYPE == type) {
    int affectedRows = taos_affected_rows(res);
    taos_free_result(res);
    return affectedRows;
H
hzcheng 已提交
1142
  }
1143

1144
  taos_free_result(res);
1145 1146
  return 0;
}
H
hzcheng 已提交
1147

1148 1149
static void appendResultBufToFile(char *resultBuf, char *resultFile)
{
1150
  FILE *fp = NULL;
1151 1152
  if (resultFile[0] != 0) {
    fp = fopen(resultFile, "at");
1153
    if (fp == NULL) {
1154 1155 1156 1157
      errorPrint(
              "%s() LN%d, failed to open result file: %s, result will not save to file\n",
              __func__, __LINE__, resultFile);
      return;
1158 1159
    }
  }
1160

1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
  fprintf(fp, "%s", resultBuf);
  tmfclose(fp);
}

static void appendResultToFile(TAOS_RES *res, char* resultFile) {
  TAOS_ROW    row = NULL;
  int         num_rows = 0;
  int         num_fields = taos_field_count(res);
  TAOS_FIELD *fields     = taos_fetch_fields(res);

1171 1172
  char* databuf = (char*) calloc(1, 100*1024*1024);
  if (databuf == NULL) {
1173 1174
    errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n",
            __func__, __LINE__);
1175 1176
    return ;
  }
H
hzcheng 已提交
1177

1178 1179
  int   totalLen = 0;
  char  temp[16000];
H
hzcheng 已提交
1180

1181
  // fetch the records row by row
1182
  while((row = taos_fetch_row(res))) {
1183
    if (totalLen >= 100*1024*1024 - 32000) {
1184
      appendResultBufToFile(databuf, resultFile);
1185 1186
      totalLen = 0;
      memset(databuf, 0, 100*1024*1024);
H
hzcheng 已提交
1187
    }
1188 1189 1190 1191 1192 1193
    num_rows++;
    int len = taos_print_row(temp, row, fields, num_fields);
    len += sprintf(temp + len, "\n");
    //printf("query result:%s\n", temp);
    memcpy(databuf + totalLen, temp, len);
    totalLen += len;
H
hzcheng 已提交
1194 1195
  }

1196 1197
  verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n",
          __func__, __LINE__, databuf, resultFile);
1198
  appendResultBufToFile(databuf, resultFile);
1199 1200
  free(databuf);
}
H
hzcheng 已提交
1201

1202
static void selectAndGetResult(
1203 1204
        threadInfo *pThreadInfo, char *command)
{
1205 1206 1207 1208 1209 1210 1211 1212 1213
  if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) {
    TAOS_RES *res = taos_query(pThreadInfo->taos, command);
    if (res == NULL || taos_errno(res) != 0) {
        errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n",
            __func__, __LINE__, command, taos_errstr(res));
        taos_free_result(res);
        return;
    }

1214 1215
    if ((strlen(pThreadInfo->fp))) {
      appendResultToFile(res, pThreadInfo->fp);
1216
    }
1217
    taos_free_result(res);
1218

1219 1220 1221
  } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
      int retCode = postProceSql(
              g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port,
1222
              command,
1223
              pThreadInfo->fp);
1224 1225 1226 1227 1228 1229 1230 1231
      if (0 != retCode) {
        printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID);
      }

  } else {
      errorPrint("%s() LN%d, unknown query mode: %s\n",
        __func__, __LINE__, g_queryInfo.queryMode);
  }
1232
}
H
hzcheng 已提交
1233

1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272
static int32_t rand_bool(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 2;
}

static int32_t rand_tinyint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 128;
}

static int32_t rand_smallint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 32767;
}

static int32_t rand_int(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor];
}

static int64_t rand_bigint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randbigint[cursor];
}

static float rand_float(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
1273
  return randfloat[cursor];
1274 1275
}

1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
#if 0
static const char charNum[] = "0123456789";

static void nonrand_string(char *, int) __attribute__ ((unused));   // reserve for debugging purpose
static void nonrand_string(char *str, int size)
{
  str[0] = 0;
  if (size > 0) {
    int n;
    for (n = 0; n < size; n++) {
      str[n] = charNum[n % 10];
    }
    str[n] = 0;
  }
}
#endif

1293
static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1294

1295
static void rand_string(char *str, int size) {
1296 1297 1298 1299
  str[0] = 0;
  if (size > 0) {
    //--size;
    int n;
1300
    for (n = 0; n < size; n++) {
1301
      int key = abs(rand_tinyint()) % (int)(sizeof(charset) - 1);
1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317
      str[n] = charset[key];
    }
    str[n] = 0;
  }
}

static double rand_double() {
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randdouble[cursor];

}

static void init_rand_data() {
  for (int i = 0; i < MAX_PREPARED_RAND; i++){
1318 1319 1320 1321
    randint[i] = (int)(taosRandom() % 65535);
    randbigint[i] = (int64_t)(taosRandom() % 2147483648);
    randfloat[i] = (float)(taosRandom() / 1000.0);
    randdouble[i] = (double)(taosRandom() / 1000000.0);
1322 1323 1324
  }
}

1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1345
static int printfInsertMeta() {
1346
    SHOW_PARSE_RESULT_START();
1347

1348
  printf("interface:                  \033[33m%s\033[0m\n",
S
Shuduo Sang 已提交
1349
          (g_args.iface==TAOSC_IFACE)?"taosc":(g_args.iface==REST_IFACE)?"rest":"stmt");
1350 1351
  printf("host:                       \033[33m%s:%u\033[0m\n",
          g_Dbs.host, g_Dbs.port);
1352 1353
  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
1354
  printf("configDir:                  \033[33m%s\033[0m\n", configDir);
1355 1356
  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
1357 1358
  printf("thread num of create table: \033[33m%d\033[0m\n",
          g_Dbs.threadCountByCreateTbl);
1359
  printf("top insert interval:        \033[33m%"PRIu64"\033[0m\n",
1360
          g_args.insert_interval);
1361
  printf("number of records per req:  \033[33m%"PRIu64"\033[0m\n",
1362
          g_args.num_of_RPR);
1363
  printf("max sql length:             \033[33m%"PRIu64"\033[0m\n",
1364
          g_args.max_sql_len);
1365 1366

  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
1367

1368 1369
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    printf("database[\033[33m%d\033[0m]:\n", i);
1370 1371
    printf("  database[%d] name:      \033[33m%s\033[0m\n",
            i, g_Dbs.db[i].dbName);
1372
    if (0 == g_Dbs.db[i].drop) {
1373 1374 1375
      printf("  drop:                  \033[33mno\033[0m\n");
    } else {
      printf("  drop:                  \033[33myes\033[0m\n");
1376 1377 1378
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
1379 1380
      printf("  blocks:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.blocks);
1381 1382
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
1383 1384
      printf("  cache:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.cache);
1385 1386
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
1387 1388
      printf("  days:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.days);
1389 1390
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
1391 1392
      printf("  keep:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.keep);
1393 1394
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
1395 1396
      printf("  replica:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.replica);
1397 1398
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
1399 1400
      printf("  update:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.update);
1401 1402
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
1403 1404
      printf("  minRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.minRows);
1405 1406
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
1407 1408
      printf("  maxRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.maxRows);
1409 1410 1411 1412 1413
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
1414 1415
      printf("  walLevel:              \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.walLevel);
1416 1417
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
1418 1419
      printf("  fsync:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.fsync);
1420 1421
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
1422 1423
      printf("  quorum:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.quorum);
1424 1425
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1426
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
1427
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1428
        printf("  precision:             \033[33m%s\033[0m\n",
1429
            g_Dbs.db[i].dbCfg.precision);
1430
      } else {
1431
        printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
1432
                g_Dbs.db[i].dbCfg.precision);
1433
        return -1;
1434 1435 1436
      }
    }

1437
    printf("  super table count:     \033[33m%"PRIu64"\033[0m\n",
1438
        g_Dbs.db[i].superTblCount);
1439 1440
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      printf("  super table[\033[33m%"PRIu64"\033[0m]:\n", j);
1441

1442
      printf("      stbName:           \033[33m%s\033[0m\n",
1443
          g_Dbs.db[i].superTbls[j].sTblName);
1444 1445 1446 1447 1448 1449 1450 1451

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
      }
1452

1453 1454 1455 1456 1457 1458 1459
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
      }
1460

1461
      printf("      childTblCount:     \033[33m%"PRId64"\033[0m\n",
1462
              g_Dbs.db[i].superTbls[j].childTblCount);
1463
      printf("      childTblPrefix:    \033[33m%s\033[0m\n",
1464
              g_Dbs.db[i].superTbls[j].childTblPrefix);
1465
      printf("      dataSource:        \033[33m%s\033[0m\n",
1466
              g_Dbs.db[i].superTbls[j].dataSource);
1467
      printf("      insertMode:        \033[33m%s\033[0m\n",
1468 1469
          (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc":
          (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt");
1470
      if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
1471
        printf("      childTblLimit:     \033[33m%"PRId64"\033[0m\n",
1472
                g_Dbs.db[i].superTbls[j].childTblLimit);
1473
      }
1474
      if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) {
1475
        printf("      childTblOffset:    \033[33m%"PRIu64"\033[0m\n",
1476
                g_Dbs.db[i].superTbls[j].childTblOffset);
1477
      }
1478
      printf("      insertRows:        \033[33m%"PRId64"\033[0m\n",
1479
              g_Dbs.db[i].superTbls[j].insertRows);
1480
/*
1481
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1482
        printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
1483
      }else {
1484
        printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
1485
      }
1486
      */
1487
      printf("      interlaceRows:     \033[33m%"PRIu64"\033[0m\n",
1488
              g_Dbs.db[i].superTbls[j].interlaceRows);
1489 1490

      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1491
        printf("      stable insert interval:   \033[33m%"PRIu64"\033[0m\n",
1492
            g_Dbs.db[i].superTbls[j].insertInterval);
1493 1494
      }

1495
      printf("      disorderRange:     \033[33m%d\033[0m\n",
1496
              g_Dbs.db[i].superTbls[j].disorderRange);
1497
      printf("      disorderRatio:     \033[33m%d\033[0m\n",
1498
              g_Dbs.db[i].superTbls[j].disorderRatio);
1499
      printf("      maxSqlLen:         \033[33m%"PRIu64"\033[0m\n",
1500
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1501
      printf("      timeStampStep:     \033[33m%"PRId64"\033[0m\n",
1502
              g_Dbs.db[i].superTbls[j].timeStampStep);
1503
      printf("      startTimestamp:    \033[33m%s\033[0m\n",
1504
              g_Dbs.db[i].superTbls[j].startTimestamp);
1505
      printf("      sampleFormat:      \033[33m%s\033[0m\n",
1506
              g_Dbs.db[i].superTbls[j].sampleFormat);
1507
      printf("      sampleFile:        \033[33m%s\033[0m\n",
1508
              g_Dbs.db[i].superTbls[j].sampleFile);
1509
      printf("      tagsFile:          \033[33m%s\033[0m\n",
1510
              g_Dbs.db[i].superTbls[j].tagsFile);
1511
      printf("      columnCount:       \033[33m%d\033[0m\n",
1512
              g_Dbs.db[i].superTbls[j].columnCount);
1513 1514
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1515
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
1516 1517 1518
                       "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "nchar", 5))) {
1519
          printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
1520 1521
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1522
        } else {
1523
          printf("column[%d]:\033[33m%s\033[0m ", k,
1524
                  g_Dbs.db[i].superTbls[j].columns[k].dataType);
1525 1526 1527
        }
      }
      printf("\n");
1528 1529

      printf("      tagCount:            \033[33m%d\033[0m\n        ",
1530
              g_Dbs.db[i].superTbls[j].tagCount);
1531 1532
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1533
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
1534 1535 1536
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1537
          printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
1538 1539
                  g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1540
        } else {
1541
          printf("tag[%d]:\033[33m%s\033[0m ", k,
1542
                  g_Dbs.db[i].superTbls[j].tags[k].dataType);
1543
        }
1544 1545 1546 1547 1548
      }
      printf("\n");
    }
    printf("\n");
  }
1549 1550

  SHOW_PARSE_RESULT_END();
1551 1552

  return 0;
1553 1554 1555
}

static void printfInsertMetaToFile(FILE* fp) {
1556 1557

  SHOW_PARSE_RESULT_START_TO_FILE(fp);
1558

1559 1560
  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
  fprintf(fp, "user:                       %s\n", g_Dbs.user);
1561
  fprintf(fp, "configDir:                  %s\n", configDir);
1562 1563 1564
  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
1565 1566
  fprintf(fp, "number of records per req:  %"PRIu64"\n", g_args.num_of_RPR);
  fprintf(fp, "max sql length:             %"PRIu64"\n", g_args.max_sql_len);
1567
  fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
1568

1569 1570
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    fprintf(fp, "database[%d]:\n", i);
1571
    fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
1572
    if (0 == g_Dbs.db[i].drop) {
1573
      fprintf(fp, "  drop:                  no\n");
1574
    }else {
1575
      fprintf(fp, "  drop:                  yes\n");
1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1615
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
1616
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1617 1618
        fprintf(fp, "  precision:             %s\n",
                g_Dbs.db[i].dbCfg.precision);
1619
      } else {
1620 1621
        fprintf(fp, "  precision error:       %s\n",
                g_Dbs.db[i].dbCfg.precision);
1622 1623 1624
      }
    }

1625 1626
    fprintf(fp, "  super table count:     %"PRIu64"\n",
            g_Dbs.db[i].superTblCount);
1627 1628
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      fprintf(fp, "  super table[%d]:\n", j);
1629

1630 1631
      fprintf(fp, "      stbName:           %s\n",
              g_Dbs.db[i].superTbls[j].sTblName);
1632 1633 1634

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "no");
1635 1636
      } else if (AUTO_CREATE_SUBTBL
              == g_Dbs.db[i].superTbls[j].autoCreateTable) {
1637 1638 1639 1640
        fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
      } else {
        fprintf(fp, "      autoCreateTable:   %s\n",  "error");
      }
1641

1642 1643
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "no");
1644 1645
      } else if (TBL_ALREADY_EXISTS
              == g_Dbs.db[i].superTbls[j].childTblExists) {
1646 1647 1648 1649
        fprintf(fp, "      childTblExists:    %s\n",  "yes");
      } else {
        fprintf(fp, "      childTblExists:    %s\n",  "error");
      }
1650

1651
      fprintf(fp, "      childTblCount:     %"PRId64"\n",
1652 1653 1654 1655 1656 1657
              g_Dbs.db[i].superTbls[j].childTblCount);
      fprintf(fp, "      childTblPrefix:    %s\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      fprintf(fp, "      dataSource:        %s\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      fprintf(fp, "      insertMode:        %s\n",
1658 1659
          (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc":
          (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt");
1660
      fprintf(fp, "      insertRows:        %"PRId64"\n",
1661
              g_Dbs.db[i].superTbls[j].insertRows);
1662
      fprintf(fp, "      interlace rows:    %"PRIu64"\n",
1663
              g_Dbs.db[i].superTbls[j].interlaceRows);
1664
      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1665
        fprintf(fp, "      stable insert interval:   %"PRIu64"\n",
1666
                g_Dbs.db[i].superTbls[j].insertInterval);
1667
      }
1668
/*
1669
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1670
        fprintf(fp, "      multiThreadWriteOneTbl:  no\n");
1671
      }else {
1672
        fprintf(fp, "      multiThreadWriteOneTbl:  yes\n");
1673
      }
1674
      */
1675
      fprintf(fp, "      interlaceRows:     %"PRIu64"\n",
1676
              g_Dbs.db[i].superTbls[j].interlaceRows);
1677 1678 1679 1680
      fprintf(fp, "      disorderRange:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      fprintf(fp, "      disorderRatio:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
1681
      fprintf(fp, "      maxSqlLen:         %"PRIu64"\n",
1682
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1683

1684 1685 1686 1687
      fprintf(fp, "      timeStampStep:     %"PRId64"\n",
              g_Dbs.db[i].superTbls[j].timeStampStep);
      fprintf(fp, "      startTimestamp:    %s\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
1688 1689 1690 1691 1692 1693
      fprintf(fp, "      sampleFormat:      %s\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      fprintf(fp, "      sampleFile:        %s\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      fprintf(fp, "      tagsFile:          %s\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
1694

1695 1696
      fprintf(fp, "      columnCount:       %d\n        ",
              g_Dbs.db[i].superTbls[j].columnCount);
1697 1698
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1699
        if ((0 == strncasecmp(
1700 1701
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "binary", strlen("binary")))
1702 1703
                || (0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
1704
                        "nchar", strlen("nchar")))) {
1705 1706 1707
          fprintf(fp, "column[%d]:%s(%d) ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1708
        } else {
1709 1710
          fprintf(fp, "column[%d]:%s ",
                  k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
1711 1712 1713
        }
      }
      fprintf(fp, "\n");
1714 1715 1716

      fprintf(fp, "      tagCount:            %d\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1717 1718
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1719
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
1720 1721 1722
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1723 1724
          fprintf(fp, "tag[%d]:%s(%d) ",
                  k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
1725
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1726 1727
        } else {
          fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
1728
        }
1729 1730 1731 1732 1733
      }
      fprintf(fp, "\n");
    }
    fprintf(fp, "\n");
  }
1734

1735
  SHOW_PARSE_RESULT_END_TO_FILE(fp);
1736 1737 1738
}

static void printfQueryMeta() {
1739

1740
  SHOW_PARSE_RESULT_START();
1741

1742
  printf("host:                    \033[33m%s:%u\033[0m\n",
1743
          g_queryInfo.host, g_queryInfo.port);
1744 1745 1746 1747
  printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
  printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

  printf("\n");
1748 1749 1750 1751

  if ((SUBSCRIBE_TEST == g_args.test_mode) || (QUERY_TEST == g_args.test_mode)) {
    printf("specified table query info:                   \n");
    printf("sqlCount:       \033[33m%"PRIu64"\033[0m\n",
1752
      g_queryInfo.specifiedQueryInfo.sqlCount);
1753 1754 1755
    if (g_queryInfo.specifiedQueryInfo.sqlCount > 0) {
      printf("specified tbl query times:\n");
      printf("                \033[33m%"PRIu64"\033[0m\n",
1756
      g_queryInfo.specifiedQueryInfo.queryTimes);
1757 1758 1759 1760 1761 1762
      printf("query interval: \033[33m%"PRIu64" ms\033[0m\n",
        g_queryInfo.specifiedQueryInfo.queryInterval);
      printf("top query times:\033[33m%"PRIu64"\033[0m\n", g_args.query_times);
      printf("concurrent:     \033[33m%"PRIu64"\033[0m\n",
      g_queryInfo.specifiedQueryInfo.concurrent);
      printf("mod:            \033[33m%s\033[0m\n",
1763
        (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync");
1764
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1765
        g_queryInfo.specifiedQueryInfo.subscribeInterval);
1766
      printf("restart:        \033[33m%d\033[0m\n",
1767
        g_queryInfo.specifiedQueryInfo.subscribeRestart);
1768
      printf("keepProgress:   \033[33m%d\033[0m\n",
1769
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
1770

1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786
      for (uint64_t i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        printf("  sql[%"PRIu64"]: \033[33m%s\033[0m\n",
            i, g_queryInfo.specifiedQueryInfo.sql[i]);
      }
      printf("\n");
    }

    printf("super table query info:\n");
    printf("sqlCount:       \033[33m%"PRIu64"\033[0m\n",
      g_queryInfo.superQueryInfo.sqlCount);

    if (g_queryInfo.superQueryInfo.sqlCount > 0) {
      printf("query interval: \033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryInterval);
      printf("threadCnt:      \033[33m%d\033[0m\n",
        g_queryInfo.superQueryInfo.threadCnt);
1787
      printf("childTblCount:  \033[33m%"PRId64"\033[0m\n",
1788 1789 1790 1791 1792 1793 1794
        g_queryInfo.superQueryInfo.childTblCount);
      printf("stable name:    \033[33m%s\033[0m\n",
        g_queryInfo.superQueryInfo.sTblName);
      printf("stb query times:\033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryTimes);

      printf("mod:            \033[33m%s\033[0m\n",
1795
        (g_queryInfo.superQueryInfo.asyncMode)?"async":"sync");
1796
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1797
        g_queryInfo.superQueryInfo.subscribeInterval);
1798
      printf("restart:        \033[33m%d\033[0m\n",
1799
        g_queryInfo.superQueryInfo.subscribeRestart);
1800
      printf("keepProgress:   \033[33m%d\033[0m\n",
1801
        g_queryInfo.superQueryInfo.subscribeKeepProgress);
1802

1803 1804 1805 1806 1807 1808
      for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
        printf("  sql[%d]: \033[33m%s\033[0m\n",
            i, g_queryInfo.superQueryInfo.sql[i]);
      }
      printf("\n");
    }
1809
  }
1810

1811
  SHOW_PARSE_RESULT_END();
1812 1813
}

1814
static char* formatTimestamp(char* buf, int64_t val, int precision) {
1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845
  time_t tt;
  if (precision == TSDB_TIME_PRECISION_MICRO) {
    tt = (time_t)(val / 1000000);
  } else {
    tt = (time_t)(val / 1000);
  }

/* comment out as it make testcases like select_with_tags.sim fail.
  but in windows, this may cause the call to localtime crash if tt < 0,
  need to find a better solution.
  if (tt < 0) {
    tt = 0;
  }
  */

#ifdef WINDOWS
  if (tt < 0) tt = 0;
#endif

  struct tm* ptm = localtime(&tt);
  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);

  if (precision == TSDB_TIME_PRECISION_MICRO) {
    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
  } else {
    sprintf(buf + pos, ".%03d", (int)(val % 1000));
  }

  return buf;
}

1846
static void xDumpFieldToFile(FILE* fp, const char* val,
1847
        TAOS_FIELD* field, int32_t length, int precision) {
1848

1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883
  if (val == NULL) {
    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
    return;
  }

  char buf[TSDB_MAX_BYTES_PER_ROW];
  switch (field->type) {
    case TSDB_DATA_TYPE_BOOL:
      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
      break;
    case TSDB_DATA_TYPE_TINYINT:
      fprintf(fp, "%d", *((int8_t *)val));
      break;
    case TSDB_DATA_TYPE_SMALLINT:
      fprintf(fp, "%d", *((int16_t *)val));
      break;
    case TSDB_DATA_TYPE_INT:
      fprintf(fp, "%d", *((int32_t *)val));
      break;
    case TSDB_DATA_TYPE_BIGINT:
      fprintf(fp, "%" PRId64, *((int64_t *)val));
      break;
    case TSDB_DATA_TYPE_FLOAT:
      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
      break;
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      memcpy(buf, val, length);
      buf[length] = 0;
      fprintf(fp, "\'%s\'", buf);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
1884
      formatTimestamp(buf, *(int64_t*)val, precision);
1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899
      fprintf(fp, "'%s'", buf);
      break;
    default:
      break;
  }
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  FILE* fp = fopen(fname, "at");
  if (fp == NULL) {
1900
    errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914
    return -1;
  }

  int num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int precision = taos_result_precision(tres);

  for (int col = 0; col < num_fields; col++) {
    if (col > 0) {
      fprintf(fp, ",");
    }
    fprintf(fp, "%s", fields[col].name);
  }
  fputc('\n', fp);
1915

1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935
  int numOfRows = 0;
  do {
    int32_t* length = taos_fetch_lengths(tres);
    for (int i = 0; i < num_fields; i++) {
      if (i > 0) {
        fputc(',', fp);
      }
      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
    }
    fputc('\n', fp);

    numOfRows++;
    row = taos_fetch_row(tres);
  } while( row != NULL);

  fclose(fp);

  return numOfRows;
}

1936 1937
static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
  TAOS_RES * res;
1938 1939
  TAOS_ROW row = NULL;
  int count = 0;
1940 1941

  res = taos_query(taos, "show databases;");
1942
  int32_t code = taos_errno(res);
1943

1944
  if (code != 0) {
1945
    errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
1946 1947 1948 1949 1950
    return -1;
  }

  TAOS_FIELD *fields = taos_fetch_fields(res);

1951
  while((row = taos_fetch_row(res)) != NULL) {
1952
    // sys database name : 'log'
1953
    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log",
1954
                fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) {
1955 1956
      continue;
    }
1957 1958 1959

    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
    if (dbInfos[count] == NULL) {
1960
      errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
1961 1962 1963
      return -1;
    }

1964
    tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
1965
            fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
1966
    formatTimestamp(dbInfos[count]->create_time,
1967 1968
            *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
            TSDB_TIME_PRECISION_MILLI);
S
Shuduo Sang 已提交
1969
    dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
1970
    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
1971 1972
    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
1973
    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
1974

1975
    tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
1976
            fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
1977 1978 1979 1980 1981 1982 1983
    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
1984
    dbInfos[count]->cachelast =
1985
      (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));
1986

1987
    tstrncpy(dbInfos[count]->precision,
1988 1989
            (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
            fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
1990
    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
1991
    tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
1992
            fields[TSDB_SHOW_DB_STATUS_INDEX].bytes);
1993

1994 1995
    count++;
    if (count > MAX_DATABASE_COUNT) {
1996
      errorPrint("%s() LN%d, The database count overflow than %d\n",
1997
         __func__, __LINE__, MAX_DATABASE_COUNT);
1998 1999 2000 2001 2002 2003 2004
      break;
    }
  }

  return count;
}

2005
static void printfDbInfoForQueryToFile(
2006
        char* filename, SDbInfo* dbInfos, int index) {
2007

2008
  if (filename[0] == 0)
2009
      return;
2010 2011 2012

  FILE *fp = fopen(filename, "at");
  if (fp == NULL) {
2013
    errorPrint( "failed to open file: %s\n", filename);
2014
    return;
2015 2016 2017 2018 2019
  }

  fprintf(fp, "================ database[%d] ================\n", index);
  fprintf(fp, "name: %s\n", dbInfos->name);
  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
S
Shuduo Sang 已提交
2020
  fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables);
2021
  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);
2022 2023
  fprintf(fp, "replica: %d\n", dbInfos->replica);
  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
2024 2025
  fprintf(fp, "days: %d\n", dbInfos->days);
  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);
2026 2027 2028 2029 2030 2031 2032
  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
  fprintf(fp, "comp: %d\n", dbInfos->comp);
2033 2034
  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);
  fprintf(fp, "precision: %s\n", dbInfos->precision);
2035
  fprintf(fp, "update: %d\n", dbInfos->update);
2036
  fprintf(fp, "status: %s\n", dbInfos->status);
2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050
  fprintf(fp, "\n");

  fclose(fp);
}

static void printfQuerySystemInfo(TAOS * taos) {
  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
  TAOS_RES* res;

  time_t t;
  struct tm* lt;
  time(&t);
  lt = localtime(&t);
2051
  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
2052 2053
          lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
          lt->tm_sec);
2054

2055 2056
  // show variables
  res = taos_query(taos, "show variables;");
2057
  //appendResultToFile(res, filename);
2058 2059 2060 2061 2062
  xDumpResultToFile(filename, res);

  // show dnodes
  res = taos_query(taos, "show dnodes;");
  xDumpResultToFile(filename, res);
2063
  //appendResultToFile(res, filename);
2064

2065 2066 2067 2068
  // show databases
  res = taos_query(taos, "show databases;");
  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
  if (dbInfos == NULL) {
2069
    errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
2070 2071 2072
    return;
  }
  int dbCount = getDbFromServer(taos, dbInfos);
2073
  if (dbCount <= 0) {
2074 2075
      free(dbInfos);
      return;
2076
  }
2077 2078

  for (int i = 0; i < dbCount; i++) {
2079
    // printf database info
2080
    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
2081

2082 2083 2084 2085
    // show db.vgroups
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);
2086

2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097
    // show db.stables
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);

    free(dbInfos[i]);
  }

  free(dbInfos);
}

2098 2099
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
        char* sqlstr, char *resultFile)
2100
{
2101
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
2102

2103
    char *url = "/rest/sql";
2104

2105 2106 2107 2108
    int bytes, sent, received, req_str_len, resp_len;
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
2109

2110
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;
2111

2112 2113 2114 2115 2116
    request_buf = malloc(req_buf_len);
    if (NULL == request_buf) {
      errorPrint("%s", "ERROR, cannot allocate memory.\n");
      exit(EXIT_FAILURE);
    }
2117

2118 2119
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};
2120

2121 2122 2123 2124 2125 2126 2127 2128
    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
2129

2130 2131 2132 2133
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);
2134

2135
    char base64_buf[INPUT_BUF_LEN];
2136
#ifdef WINDOWS
2137
    WSADATA wsaData;
2138
    WSAStartup(MAKEWORD(2, 2), &wsaData);
2139 2140
    SOCKET sockfd;
#else
2141
    int sockfd;
2142
#endif
2143 2144
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (sockfd < 0) {
2145
#ifdef WINDOWS
2146
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
2147
#endif
2148 2149 2150 2151
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
        free(request_buf);
        ERROR_EXIT("ERROR opening socket");
    }
2152

2153
    int retConn = connect(sockfd, (struct sockaddr *)pServAddr, sizeof(struct sockaddr));
2154 2155 2156 2157 2158
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
        free(request_buf);
        ERROR_EXIT("ERROR connecting");
    }
2159

2160
    memset(base64_buf, 0, INPUT_BUF_LEN);
2161

2162 2163 2164 2165 2166 2167 2168 2169
    for (int n = 0, m = 0; n < userpass_buf_len;) {
      uint32_t oct_a = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t oct_b = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t oct_c = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;
2170

2171 2172 2173 2174 2175
      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }
2176

2177 2178
    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';
2179

2180 2181 2182
    debugPrint("%s() LN%d: auth string base64 encoded: %s\n",
            __func__, __LINE__, base64_buf);
    char *auth = base64_buf;
2183

2184 2185 2186 2187 2188 2189 2190 2191 2192
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
2193

2194 2195 2196
    req_str_len = strlen(request_buf);
    sent = 0;
    do {
2197
#ifdef WINDOWS
2198
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
2199
#else
2200
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
2201
#endif
2202 2203 2204 2205 2206 2207
        if (bytes < 0)
            ERROR_EXIT("ERROR writing message to socket");
        if (bytes == 0)
            break;
        sent+=bytes;
    } while(sent < req_str_len);
2208

2209 2210 2211 2212
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
    received = 0;
    do {
2213
#ifdef WINDOWS
2214
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
2215
#else
2216
        bytes = read(sockfd, response_buf + received, resp_len - received);
2217
#endif
2218 2219 2220 2221 2222 2223 2224 2225
        if (bytes < 0) {
            free(request_buf);
            ERROR_EXIT("ERROR reading response from socket");
        }
        if (bytes == 0)
            break;
        received += bytes;
    } while(received < resp_len);
2226

2227 2228 2229 2230
    if (received == resp_len) {
        free(request_buf);
        ERROR_EXIT("ERROR storing complete response from socket");
    }
H
Haojun Liao 已提交
2231

2232 2233
    response_buf[RESP_BUF_LEN - 1] = '\0';
    printf("Response:\n%s\n", response_buf);
H
Haojun Liao 已提交
2234

2235 2236 2237 2238
    if (resultFile) {
       appendResultBufToFile(response_buf, resultFile);
    }

2239
    free(request_buf);
2240
#ifdef WINDOWS
2241
    closesocket(sockfd);
2242 2243
    WSACleanup();
#else
2244
    close(sockfd);
2245
#endif
2246

2247
    return 0;
2248 2249
}

2250
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
2251 2252
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
2253
    errorPrint("%s() LN%d, calloc failed! size:%d\n",
2254
        __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
2255 2256
    return NULL;
  }
2257

2258
  int    dataLen = 0;
2259
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2260
          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);
2261

2262 2263 2264
  return dataBuf;
}

2265
static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) {
2266 2267 2268 2269 2270 2271 2272 2273 2274
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
    printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
    return NULL;
  }

  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
  for (int i = 0; i < stbInfo->tagCount; i++) {
2275
    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
2276
            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
2277
      if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
2278
        printf("binary or nchar length overflow, max size:%u\n",
2279
                (uint32_t)TSDB_MAX_BINARY_LEN);
2280 2281 2282
        tmfree(dataBuf);
        return NULL;
      }
2283

2284 2285
      int tagBufLen = stbInfo->tags[i].dataLen + 1;
      char* buf = (char*)calloc(tagBufLen, 1);
2286 2287 2288 2289 2290
      if (NULL == buf) {
        printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
        tmfree(dataBuf);
        return NULL;
      }
2291 2292 2293 2294 2295 2296 2297

      if (tableSeq % 2) {
        tstrncpy(buf, "beijing", tagBufLen);
      } else {
        tstrncpy(buf, "shanghai", tagBufLen);
      }
      //rand_string(buf, stbInfo->tags[i].dataLen);
2298
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2299
              "\'%s\', ", buf);
2300
      tmfree(buf);
2301
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2302
                "int", strlen("int"))) {
2303
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2304
              "%d, ", tableSeq);
2305
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2306
                "bigint", strlen("bigint"))) {
2307
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2308
              "%"PRId64", ", rand_bigint());
2309
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2310
                "float", strlen("float"))) {
2311
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2312
              "%f, ", rand_float());
2313
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2314
                "double", strlen("double"))) {
2315
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2316
              "%f, ", rand_double());
2317
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2318
                "smallint", strlen("smallint"))) {
2319
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2320
              "%d, ", rand_smallint());
2321
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2322
                "tinyint", strlen("tinyint"))) {
2323
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2324
              "%d, ", rand_tinyint());
2325
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2326
                "bool", strlen("bool"))) {
2327
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2328
              "%d, ", rand_bool());
2329
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2330
                "timestamp", strlen("timestamp"))) {
2331
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2332
              "%"PRId64", ", rand_bigint());
2333 2334 2335 2336 2337 2338
    }  else {
      printf("No support data type: %s\n", stbInfo->tags[i].dataType);
      tmfree(dataBuf);
      return NULL;
    }
  }
2339

2340
  dataLen -= 2;
2341
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");
2342 2343 2344
  return dataBuf;
}

2345
static int calcRowLen(SSuperTable*  superTbls) {
2346 2347
  int colIndex;
  int  lenOfOneRow = 0;
2348

2349 2350
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2351

2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
2368
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2369
      lenOfOneRow += 42;
2370
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
2384

2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
2401
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2402 2403 2404 2405 2406 2407 2408 2409
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2410

2411 2412 2413 2414
  return 0;
}


2415
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
2416
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
2417
        int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) {
2418

2419
  char command[BUFFER_SIZE] = "\0";
2420 2421
  char limitBuf[100] = "\0";

2422
  TAOS_RES * res;
2423 2424 2425
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2426 2427

  if (offset >= 0) {
2428
    snprintf(limitBuf, 100, " limit %"PRId64" offset %"PRIu64"",
2429
            limit, offset);
2430 2431
  }

2432
  //get all child table name use cmd: select tbname from superTblName;
2433
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s",
2434
          dbName, sTblName, limitBuf);
2435

2436
  res = taos_query(taos, command);
2437 2438 2439 2440
  int32_t code = taos_errno(res);
  if (code != 0) {
    taos_free_result(res);
    taos_close(taos);
2441
    errorPrint("%s() LN%d, failed to run command %s\n",
2442
           __func__, __LINE__, command);
2443 2444 2445
    exit(-1);
  }

2446 2447
  int64_t childTblCount = (limit < 0)?10000:limit;
  int64_t count = 0;
2448 2449 2450
  if (childTblName == NULL) {
    childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (NULL ==  childTblName) {
2451 2452 2453 2454
    taos_free_result(res);
        taos_close(taos);
        errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__);
        exit(-1);
2455 2456 2457
    }
  }

2458
  char* pTblName = childTblName;
2459
  while((row = taos_fetch_row(res)) != NULL) {
2460
    int32_t* len = taos_fetch_lengths(res);
2461
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2462 2463 2464
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2465
      char *tmp = realloc(childTblName,
2466
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2467 2468 2469
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2470
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
2471
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2472 2473
      } else {
        // exit, if allocate more memory failed
2474
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
2475
               __func__, __LINE__, dbName, sTblName);
2476 2477 2478 2479 2480 2481
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2482
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2483
  }
2484

2485 2486 2487 2488 2489 2490 2491
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2492
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
2493
        char* sTblName, char** childTblNameOfSuperTbl,
2494
        int64_t* childTblCountOfSuperTbl) {
2495

2496 2497
    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
2498
            -1, 0);
2499 2500
}

2501
static int getSuperTableFromServer(TAOS * taos, char* dbName,
2502
        SSuperTable*  superTbls) {
2503

2504
  char command[BUFFER_SIZE] = "\0";
2505
  TAOS_RES * res;
2506 2507
  TAOS_ROW row = NULL;
  int count = 0;
2508

2509
  //get schema use cmd: describe superTblName;
2510
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2511
  res = taos_query(taos, command);
2512 2513 2514
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2515
    taos_free_result(res);
2516 2517 2518 2519 2520 2521
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
2522
  while((row = taos_fetch_row(res)) != NULL) {
2523 2524 2525
    if (0 == count) {
      count++;
      continue;
2526
    }
2527 2528

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2529
      tstrncpy(superTbls->tags[tagIndex].field,
2530 2531
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
2532
      tstrncpy(superTbls->tags[tagIndex].dataType,
2533 2534
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
2535 2536 2537
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
2538 2539
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2540
      tagIndex++;
2541
    } else {
2542
      tstrncpy(superTbls->columns[columnIndex].field,
2543 2544
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
2545
      tstrncpy(superTbls->columns[columnIndex].dataType,
2546 2547
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
2548 2549 2550
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
2551 2552
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2564
/*
2565
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
2566
    //get all child table name use cmd: select tbname from superTblName;
2567 2568 2569
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2570
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2571 2572
      return -1;
    }
2573
    getAllChildNameOfSuperTable(taos, dbName,
2574 2575 2576
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2577
  }
2578
  */
2579 2580 2581
  return 0;
}

H
Haojun Liao 已提交
2582
static int createSuperTable(
2583 2584
        TAOS * taos, char* dbName,
        SSuperTable*  superTbl) {
H
Haojun Liao 已提交
2585

2586
  char command[BUFFER_SIZE] = "\0";
2587

2588 2589 2590 2591 2592
  char cols[STRING_LEN] = "\0";
  int colIndex;
  int len = 0;

  int  lenOfOneRow = 0;
2593 2594 2595

  if (superTbl->columnCount == 0) {
    errorPrint("%s() LN%d, super table column count is %d\n",
2596
            __func__, __LINE__, superTbl->columnCount);
2597 2598 2599 2600 2601
    return -1;
  }

  for (colIndex = 0; colIndex < superTbl->columnCount; colIndex++) {
    char* dataType = superTbl->columns[colIndex].dataType;
2602

2603
    if (strcasecmp(dataType, "BINARY") == 0) {
2604
      len += snprintf(cols + len, STRING_LEN - len,
2605 2606
          ", col%d %s(%d)", colIndex, "BINARY",
          superTbl->columns[colIndex].dataLen);
2607
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2608
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
2609
      len += snprintf(cols + len, STRING_LEN - len,
2610 2611
          ", col%d %s(%d)", colIndex, "NCHAR",
          superTbl->columns[colIndex].dataLen);
2612
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
      lenOfOneRow += 22;
2631
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2632 2633
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
      lenOfOneRow += 42;
2634
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2635 2636 2637 2638
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
      lenOfOneRow += 21;
    } else {
      taos_close(taos);
2639
      errorPrint("%s() LN%d, config error data type : %s\n",
2640
         __func__, __LINE__, dataType);
2641 2642 2643 2644
      exit(-1);
    }
  }

2645 2646
  superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp
  //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbl[j].sTblName, g_Dbs.db[i].superTbl[j].columnCount, lenOfOneRow);
2647 2648

  // save for creating child table
2649 2650 2651
  superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1);
  if (NULL == superTbl->colsOfCreateChildTable) {
    errorPrint("%s() LN%d, Failed when calloc, size:%d",
2652
           __func__, __LINE__, len+1);
2653 2654 2655 2656
    taos_close(taos);
    exit(-1);
  }

2657
  snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
2658
  verbosePrint("%s() LN%d: %s\n",
2659
      __func__, __LINE__, superTbl->colsOfCreateChildTable);
2660

2661 2662
  if (superTbl->tagCount == 0) {
    errorPrint("%s() LN%d, super table tag count is %d\n",
2663
            __func__, __LINE__, superTbl->tagCount);
2664 2665
    return -1;
  }
2666

2667 2668 2669
  char tags[STRING_LEN] = "\0";
  int tagIndex;
  len = 0;
2670

2671 2672 2673 2674
  int lenOfTagOfOneRow = 0;
  len += snprintf(tags + len, STRING_LEN - len, "(");
  for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) {
    char* dataType = superTbl->tags[tagIndex].dataType;
2675

2676 2677
    if (strcasecmp(dataType, "BINARY") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
2678
              "BINARY", superTbl->tags[tagIndex].dataLen);
2679 2680 2681
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
2682
              "NCHAR", superTbl->tags[tagIndex].dataLen);
2683 2684 2685
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2686
              "INT");
2687 2688 2689
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2690
              "BIGINT");
2691 2692 2693
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2694
              "SMALLINT");
2695 2696 2697
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2698
              "TINYINT");
2699 2700 2701
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2702
              "BOOL");
2703 2704 2705
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2706
              "FLOAT");
2707 2708 2709
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2710
              "DOUBLE");
2711 2712 2713
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42;
    } else {
      taos_close(taos);
2714
      errorPrint("%s() LN%d, config error tag type : %s\n",
2715
         __func__, __LINE__, dataType);
2716
      exit(-1);
2717 2718
    }
  }
2719 2720 2721 2722 2723 2724 2725

  len -= 2;
  len += snprintf(tags + len, STRING_LEN - len, ")");

  superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow;

  snprintf(command, BUFFER_SIZE,
2726 2727
          "create table if not exists %s.%s (ts timestamp%s) tags %s",
          dbName, superTbl->sTblName, cols, tags);
2728 2729 2730
  verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command);

  if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2731 2732 2733
      errorPrint( "create supertable %s failed!\n\n",
              superTbl->sTblName);
      return -1;
2734 2735
  }
  debugPrint("create supertable %s success!\n\n", superTbl->sTblName);
2736 2737 2738
  return 0;
}

2739
static int createDatabasesAndStables() {
2740 2741 2742 2743
  TAOS * taos = NULL;
  int    ret = 0;
  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
  if (taos == NULL) {
2744
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2745
    return -1;
2746 2747 2748
  }
  char command[BUFFER_SIZE] = "\0";

2749
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2750 2751
    if (g_Dbs.db[i].drop) {
      sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
2752
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);
2753
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2754 2755 2756 2757
        taos_close(taos);
        return -1;
      }

2758
      int dataLen = 0;
2759
      dataLen += snprintf(command + dataLen,
2760
          BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2761

2762 2763
      if (g_Dbs.db[i].dbCfg.blocks > 0) {
        dataLen += snprintf(command + dataLen,
2764
            BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
2765 2766 2767
      }
      if (g_Dbs.db[i].dbCfg.cache > 0) {
        dataLen += snprintf(command + dataLen,
2768
            BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
2769 2770 2771
      }
      if (g_Dbs.db[i].dbCfg.days > 0) {
        dataLen += snprintf(command + dataLen,
2772
            BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
2773 2774 2775
      }
      if (g_Dbs.db[i].dbCfg.keep > 0) {
        dataLen += snprintf(command + dataLen,
2776
            BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
2777 2778 2779
      }
      if (g_Dbs.db[i].dbCfg.quorum > 1) {
        dataLen += snprintf(command + dataLen,
2780
            BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
2781 2782 2783
      }
      if (g_Dbs.db[i].dbCfg.replica > 0) {
        dataLen += snprintf(command + dataLen,
2784
            BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
2785 2786 2787
      }
      if (g_Dbs.db[i].dbCfg.update > 0) {
        dataLen += snprintf(command + dataLen,
2788
            BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
2789 2790 2791 2792 2793 2794 2795
      }
      //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
      //  dataLen += snprintf(command + dataLen,
      //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
      //}
      if (g_Dbs.db[i].dbCfg.minRows > 0) {
        dataLen += snprintf(command + dataLen,
2796
            BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
2797 2798 2799
      }
      if (g_Dbs.db[i].dbCfg.maxRows > 0) {
        dataLen += snprintf(command + dataLen,
2800
            BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
2801 2802 2803
      }
      if (g_Dbs.db[i].dbCfg.comp > 0) {
        dataLen += snprintf(command + dataLen,
2804
            BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
2805 2806 2807
      }
      if (g_Dbs.db[i].dbCfg.walLevel > 0) {
        dataLen += snprintf(command + dataLen,
2808
            BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
2809 2810 2811
      }
      if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
        dataLen += snprintf(command + dataLen,
2812
            BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
2813 2814 2815
      }
      if (g_Dbs.db[i].dbCfg.fsync > 0) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2816
                " fsync %d", g_Dbs.db[i].dbCfg.fsync);
2817 2818
      }
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
2819 2820
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                      "us", strlen("us")))) {
2821
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2822
                " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
2823 2824 2825 2826 2827 2828 2829 2830 2831
      }

      debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
        taos_close(taos);
        errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
        return -1;
      }
      printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
2832 2833
    }

2834
    debugPrint("%s() LN%d supertbl count:%"PRIu64"\n",
2835
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
2836 2837 2838

    int validStbCount = 0;

S
Shuduo Sang 已提交
2839
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
2840 2841 2842 2843 2844 2845 2846
      sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
              g_Dbs.db[i].superTbls[j].sTblName);
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);

      ret = queryDbExec(taos, command, NO_INSERT_TYPE, true);

      if ((ret != 0) || (g_Dbs.db[i].drop)) {
2847
        ret = createSuperTable(taos, g_Dbs.db[i].dbName,
2848
                &g_Dbs.db[i].superTbls[j]);
2849

2850
        if (0 != ret) {
S
Shuduo Sang 已提交
2851
          errorPrint("create super table %"PRIu64" failed!\n\n", j);
2852
          continue;
2853 2854 2855
        }
      }

2856
      ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
2857
                &g_Dbs.db[i].superTbls[j]);
2858 2859
      if (0 != ret) {
        errorPrint("\nget super table %s.%s info failed!\n\n",
2860
                g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName);
2861
        continue;
2862
      }
2863 2864

      validStbCount ++;
2865
    }
2866 2867

    g_Dbs.db[i].superTblCount = validStbCount;
2868 2869 2870 2871 2872 2873
  }

  taos_close(taos);
  return 0;
}

2874 2875
static void* createTable(void *sarg)
{
2876 2877
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
2878

S
Shuduo Sang 已提交
2879
  uint64_t  lastPrintTime = taosGetTimestampMs();
2880

2881
  int buff_len;
2882
  buff_len = BUFFER_SIZE / 8;
2883

2884
  char *buffer = calloc(buff_len, 1);
2885
  if (buffer == NULL) {
2886
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
2887 2888
    exit(-1);
  }
2889 2890 2891

  int len = 0;
  int batchNum = 0;
2892

2893
  verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n",
2894 2895
          __func__, __LINE__,
          pThreadInfo->start_table_from, pThreadInfo->end_table_to);
2896

2897
  for (uint64_t i = pThreadInfo->start_table_from;
2898
          i <= pThreadInfo->end_table_to; i++) {
2899
    if (0 == g_Dbs.use_metric) {
2900
      snprintf(buffer, buff_len,
2901
              "create table if not exists %s.%s%"PRIu64" %s;",
2902 2903 2904
              pThreadInfo->db_name,
              g_args.tb_prefix, i,
              pThreadInfo->cols);
2905
    } else {
2906 2907
      if (superTblInfo == NULL) {
        errorPrint("%s() LN%d, use metric, but super table info is NULL\n",
2908
                  __func__, __LINE__);
2909
        free(buffer);
2910 2911 2912 2913 2914 2915
        exit(-1);
      } else {
        if (0 == len) {
          batchNum = 0;
          memset(buffer, 0, buff_len);
          len += snprintf(buffer + len,
2916
                  buff_len - len, "create table ");
2917 2918 2919
        }
        char* tagsValBuf = NULL;
        if (0 == superTblInfo->tagSource) {
2920
          tagsValBuf = generateTagVaulesForStb(superTblInfo, i);
2921 2922
        } else {
          tagsValBuf = getTagValueFromTagSample(
2923 2924
                  superTblInfo,
                  i % superTblInfo->tagSampleCount);
2925 2926 2927 2928 2929 2930
        }
        if (NULL == tagsValBuf) {
          free(buffer);
          return NULL;
        }
        len += snprintf(buffer + len,
2931
                buff_len - len,
2932
                "if not exists %s.%s%"PRIu64" using %s.%s tags %s ",
2933 2934 2935
                pThreadInfo->db_name, superTblInfo->childTblPrefix,
                i, pThreadInfo->db_name,
                superTblInfo->sTblName, tagsValBuf);
2936 2937 2938
        free(tagsValBuf);
        batchNum++;
        if ((batchNum < superTblInfo->batchCreateTableNum)
2939 2940
                && ((buff_len - len)
                    >= (superTblInfo->lenOfTagOfOneRow + 256))) {
2941 2942
          continue;
        }
2943 2944 2945 2946
      }
    }

    len = 0;
2947
    verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
2948
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)){
2949
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2950
      free(buffer);
2951 2952 2953
      return NULL;
    }

S
Shuduo Sang 已提交
2954
    uint64_t  currentPrintTime = taosGetTimestampMs();
2955
    if (currentPrintTime - lastPrintTime > 30*1000) {
2956
      printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n",
2957
              pThreadInfo->threadID, pThreadInfo->start_table_from, i);
2958 2959 2960
      lastPrintTime = currentPrintTime;
    }
  }
2961

2962
  if (0 != len) {
2963
    verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer);
2964
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)) {
2965
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2966
    }
2967
  }
2968

2969 2970 2971 2972
  free(buffer);
  return NULL;
}

2973
static int startMultiThreadCreateChildTable(
S
Shuduo Sang 已提交
2974
        char* cols, int threads, uint64_t startFrom, int64_t ntables,
2975
        char* db_name, SSuperTable* superTblInfo) {
2976

2977
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
2978
  threadInfo *infos = calloc(1, threads * sizeof(threadInfo));
2979 2980 2981 2982 2983 2984 2985 2986 2987 2988

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

S
Shuduo Sang 已提交
2989
  int64_t a = ntables / threads;
2990 2991 2992 2993 2994
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

S
Shuduo Sang 已提交
2995
  int64_t b = 0;
2996
  b = ntables % threads;
2997

S
Shuduo Sang 已提交
2998
  for (int i = 0; i < threads; i++) {
2999 3000
    threadInfo *t_info = infos + i;
    t_info->threadID = i;
H
Hui Li 已提交
3001
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
3002
    t_info->superTblInfo = superTblInfo;
3003
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
3004
    t_info->taos = taos_connect(
3005 3006 3007 3008 3009
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
3010
    if (t_info->taos == NULL) {
3011
      errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n",
3012
         __func__, __LINE__, taos_errstr(NULL));
3013
      free(pids);
3014
      free(infos);
3015 3016
      return -1;
    }
3017 3018 3019 3020 3021

    t_info->start_table_from = startFrom;
    t_info->ntables = i<b?a+1:a;
    t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
    startFrom = t_info->end_table_to + 1;
3022
    t_info->use_metric = true;
3023
    t_info->cols = cols;
3024
    t_info->minDelay = UINT64_MAX;
3025 3026
    pthread_create(pids + i, NULL, createTable, t_info);
  }
3027

3028 3029 3030 3031 3032 3033 3034 3035 3036 3037
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
    threadInfo *t_info = infos + i;
    taos_close(t_info->taos);
  }

  free(pids);
3038
  free(infos);
3039 3040

  return 0;
3041 3042
}

3043
static void createChildTables() {
3044 3045
    char tblColsBuf[MAX_SQL_SIZE];
    int len;
3046

3047
  for (int i = 0; i < g_Dbs.dbCount; i++) {
3048 3049
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
3050
          // with super table
S
Shuduo Sang 已提交
3051
        for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
3052
          if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
3053
                || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
3054 3055
            continue;
          }
3056

3057
          verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
3058
                  g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
S
Shuduo Sang 已提交
3059
          uint64_t startFrom = 0;
3060 3061
          g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;

3062
          verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n",
3063
                  __func__, __LINE__, g_totalChildTables, startFrom);
3064
          startMultiThreadCreateChildTable(
3065 3066 3067 3068 3069
                g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
                g_Dbs.threadCountByCreateTbl,
                startFrom,
                g_Dbs.db[i].superTbls[j].childTblCount,
                g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
3070
        }
3071 3072
      }
    } else {
3073 3074
      // normal table
      len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
H
Haojun Liao 已提交
3075
      for (int j = 0; j < g_args.num_of_CPR; j++) {
3076 3077 3078 3079 3080 3081 3082 3083 3084 3085
          if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
                  || (strncasecmp(g_args.datatype[j],
                      "NCHAR", strlen("NCHAR")) == 0)) {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                      ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary);
          } else {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                      ", COL%d %s", j, g_args.datatype[j]);
          }
          len = strlen(tblColsBuf);
3086
      }
3087

3088
      snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");
3089

3090
      verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n",
3091 3092
              __func__, __LINE__,
              g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
3093
      startMultiThreadCreateChildTable(
3094 3095 3096 3097 3098 3099
            tblColsBuf,
            g_Dbs.threadCountByCreateTbl,
            0,
            g_args.num_of_tables,
            g_Dbs.db[i].dbName,
            NULL);
3100
    }
3101 3102 3103 3104 3105 3106
  }
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3107
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
3108 3109 3110
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
3111

3112 3113
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
3114
    printf("Failed to open tags file: %s, reason:%s\n",
3115
            superTblInfo->tagsFile, strerror(errno));
3116 3117 3118 3119 3120 3121 3122
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
3123

3124 3125 3126 3127 3128 3129 3130 3131 3132
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

3133
  while((readLen = tgetline(&line, &n, fp)) != -1) {
3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
3146
      char *tmp = realloc(tagDataBuf,
3147
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
3148 3149 3150
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
3151
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
3152
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

3172
#if 0
3173 3174 3175 3176
int readSampleFromJsonFileToMem(SSuperTable  * superTblInfo) {
  // TODO
  return 0;
}
3177
#endif
3178 3179 3180 3181

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3182
static int readSampleFromCsvFileToMem(
3183
        SSuperTable* superTblInfo) {
3184 3185 3186 3187
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
3188

3189 3190
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
3191 3192 3193
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
              superTblInfo->sampleFile, strerror(errno));
      return -1;
3194
  }
3195

3196
  assert(superTblInfo->sampleDataBuf);
3197
  memset(superTblInfo->sampleDataBuf, 0,
3198
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
3199
  while(1) {
3200
    readLen = tgetline(&line, &n, fp);
3201 3202
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
3203
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
3204
                superTblInfo->sampleFile, strerror(errno));
3205
        fclose(fp);
3206 3207 3208 3209
        return -1;
      }
      continue;
    }
3210

3211 3212 3213 3214 3215 3216 3217 3218 3219
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
3220
      printf("sample row len[%d] overflow define schema len[%"PRIu64"], so discard this row\n",
3221
              (int32_t)readLen, superTblInfo->lenOfOneRow);
3222 3223 3224
      continue;
    }

3225
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
3226
          line, readLen);
3227 3228 3229 3230 3231 3232 3233
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

3234
  fclose(fp);
3235 3236 3237 3238
  tmfree(line);
  return 0;
}

3239
static bool getColumnAndTagTypeFromInsertJsonFile(
3240
        cJSON* stbInfo, SSuperTable* superTbls) {
3241
  bool  ret = false;
3242

3243
  // columns
3244 3245
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
3246
    printf("ERROR: failed to read json, columns not found\n");
3247 3248 3249 3250 3251 3252
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
3253

3254
  int columnSize = cJSON_GetArraySize(columns);
3255
  if ((columnSize + 1/* ts */) > MAX_COLUMN_COUNT) {
3256
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
3257
            __func__, __LINE__, MAX_COLUMN_COUNT);
3258 3259 3260 3261 3262 3263
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
3264

3265
  //superTbls->columnCount = columnSize;
3266 3267 3268 3269 3270 3271 3272
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
3273
      count = countObj->valueint;
3274
    } else if (countObj && countObj->type != cJSON_Number) {
3275
      errorPrint("%s() LN%d, failed to read json, column count not found\n",
3276
          __func__, __LINE__);
3277 3278 3279 3280 3281
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3282
    // column info
3283 3284
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
3285 3286 3287
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d: failed to read json, column type not found\n",
3288
          __func__, __LINE__);
3289 3290
      goto PARSE_OVER;
    }
3291 3292
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3293

3294 3295
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3296
      columnCase.dataLen = dataLen->valueint;
3297
    } else if (dataLen && dataLen->type != cJSON_Number) {
3298
      debugPrint("%s() LN%d: failed to read json, column len not found\n",
3299
          __func__, __LINE__);
3300 3301 3302 3303
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
3304

3305
    for (int n = 0; n < count; ++n) {
3306
      tstrncpy(superTbls->columns[index].dataType,
3307
              columnCase.dataType, MAX_TB_NAME_SIZE);
3308
      superTbls->columns[index].dataLen = columnCase.dataLen;
3309 3310
      index++;
    }
3311
  }
3312

3313 3314
  if ((index + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n",
3315
            __func__, __LINE__, MAX_COLUMN_COUNT);
3316 3317 3318
    goto PARSE_OVER;
  }

3319
  superTbls->columnCount = index;
3320

3321 3322
  count = 1;
  index = 0;
3323
  // tags
3324 3325
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
3326
    errorPrint("%s() LN%d, failed to read json, tags not found\n",
3327
        __func__, __LINE__);
3328 3329
    goto PARSE_OVER;
  }
3330

3331 3332
  int tagSize = cJSON_GetArraySize(tags);
  if (tagSize > MAX_TAG_COUNT) {
3333
    errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n",
3334
        __func__, __LINE__, MAX_TAG_COUNT);
3335 3336
    goto PARSE_OVER;
  }
3337 3338

  //superTbls->tagCount = tagSize;
3339 3340 3341
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
3342

3343 3344 3345
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
3346
      count = countObj->valueint;
3347
    } else if (countObj && countObj->type != cJSON_Number) {
3348
      printf("ERROR: failed to read json, column count not found\n");
3349 3350 3351 3352 3353
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3354
    // column info
3355 3356
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
3357 3358 3359
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d, failed to read json, tag type not found\n",
3360
          __func__, __LINE__);
3361 3362
      goto PARSE_OVER;
    }
3363
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3364

3365 3366
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3367
      columnCase.dataLen = dataLen->valueint;
3368
    } else if (dataLen && dataLen->type != cJSON_Number) {
3369
      errorPrint("%s() LN%d, failed to read json, column len not found\n",
3370
          __func__, __LINE__);
3371 3372 3373
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
3374 3375
    }

3376
    for (int n = 0; n < count; ++n) {
3377
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType,
3378
          MAX_TB_NAME_SIZE);
3379
      superTbls->tags[index].dataLen = columnCase.dataLen;
3380 3381
      index++;
    }
3382
  }
3383 3384

  if (index > MAX_TAG_COUNT) {
3385
    errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n",
3386
        __func__, __LINE__, MAX_TAG_COUNT);
3387 3388 3389
    goto PARSE_OVER;
  }

3390 3391
  superTbls->tagCount = index;

3392 3393
  if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n",
3394
        __func__, __LINE__, MAX_COLUMN_COUNT);
3395 3396
    goto PARSE_OVER;
  }
3397 3398
  ret = true;

3399
PARSE_OVER:
3400 3401 3402 3403 3404 3405 3406 3407
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3408
    tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3409 3410 3411 3412
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3413
    tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE);
3414
  } else if (!host) {
3415
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3416
  } else {
3417
    printf("ERROR: failed to read json, host not found\n");
3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_Dbs.port = port->valueint;
  } else if (!port) {
    g_Dbs.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3430
    tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE);
3431
  } else if (!user) {
3432
    tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE);
3433 3434 3435 3436
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3437
    tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE);
3438
  } else if (!password) {
3439
    tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE);
3440 3441 3442 3443
  }

  cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
  if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
3444
    tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
3445
  } else if (!resultfile) {
3446
    tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
3447 3448 3449 3450 3451 3452 3453 3454
  }

  cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
  if (threads && threads->type == cJSON_Number) {
    g_Dbs.threadCount = threads->valueint;
  } else if (!threads) {
    g_Dbs.threadCount = 1;
  } else {
3455
    printf("ERROR: failed to read json, threads not found\n");
3456
    goto PARSE_OVER;
3457 3458
  }

3459 3460 3461 3462
  cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
  if (threads2 && threads2->type == cJSON_Number) {
    g_Dbs.threadCountByCreateTbl = threads2->valueint;
  } else if (!threads2) {
sangshuduo's avatar
sangshuduo 已提交
3463
    g_Dbs.threadCountByCreateTbl = 1;
3464
  } else {
3465
    errorPrint("%s() LN%d, failed to read json, threads2 not found\n",
3466
            __func__, __LINE__);
3467
    goto PARSE_OVER;
3468
  }
3469

3470 3471
  cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
  if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
3472 3473 3474 3475 3476
    if (gInsertInterval->valueint <0) {
      errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n",
            __func__, __LINE__);
      goto PARSE_OVER;
    }
3477 3478 3479 3480
    g_args.insert_interval = gInsertInterval->valueint;
  } else if (!gInsertInterval) {
    g_args.insert_interval = 0;
  } else {
3481
    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
3482
        __func__, __LINE__);
3483 3484
    goto PARSE_OVER;
  }
3485

3486 3487
  cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows");
  if (interlaceRows && interlaceRows->type == cJSON_Number) {
3488 3489 3490 3491 3492 3493
    if (interlaceRows->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;

    }
3494 3495
    g_args.interlace_rows = interlaceRows->valueint;
  } else if (!interlaceRows) {
3496
    g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3497
  } else {
3498
    errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
3499
        __func__, __LINE__);
3500
    goto PARSE_OVER;
3501
  }
3502

3503 3504
  cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
  if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
3505 3506 3507 3508 3509
    if (maxSqlLen->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
    }
3510 3511
    g_args.max_sql_len = maxSqlLen->valueint;
  } else if (!maxSqlLen) {
sangshuduo's avatar
sangshuduo 已提交
3512
    g_args.max_sql_len = (1024*1024);
3513
  } else {
3514
    errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
3515
        __func__, __LINE__);
3516 3517 3518 3519 3520
    goto PARSE_OVER;
  }

  cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
  if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
3521
    if (numRecPerReq->valueint <= 0) {
3522 3523 3524
      errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
3525
    } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) {
3526 3527 3528 3529 3530
      printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n",
              numRecPerReq->valueint, MAX_RECORDS_PER_REQ);
      printf("        number of records per request value will be set to %d\n\n",
              MAX_RECORDS_PER_REQ);
      prompt();
3531
      numRecPerReq->valueint = MAX_RECORDS_PER_REQ;
3532
    }
3533 3534
    g_args.num_of_RPR = numRecPerReq->valueint;
  } else if (!numRecPerReq) {
3535
    g_args.num_of_RPR = MAX_RECORDS_PER_REQ;
3536
  } else {
3537
    errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n",
3538
        __func__, __LINE__);
3539 3540 3541
    goto PARSE_OVER;
  }

3542
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3543
  if (answerPrompt
3544 3545
          && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3546 3547 3548 3549 3550 3551 3552 3553
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
    } else {
      g_args.answer_yes = false;
    }
  } else if (!answerPrompt) {
3554
    g_args.answer_yes = true;   // default is no, mean answer_yes.
3555
  } else {
3556
    errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n");
3557
    goto PARSE_OVER;
3558
  }
3559

3560 3561 3562 3563 3564 3565 3566 3567 3568 3569
  // rows per table need be less than insert batch
  if (g_args.interlace_rows > g_args.num_of_RPR) {
      printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
              g_args.interlace_rows, g_args.num_of_RPR);
      printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
              g_args.num_of_RPR);
      prompt();
      g_args.interlace_rows = g_args.num_of_RPR;
  }

3570 3571
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (!dbs || dbs->type != cJSON_Array) {
3572
    printf("ERROR: failed to read json, databases not found\n");
3573 3574 3575 3576 3577
    goto PARSE_OVER;
  }

  int dbSize = cJSON_GetArraySize(dbs);
  if (dbSize > MAX_DB_COUNT) {
3578
    errorPrint(
3579 3580
            "ERROR: failed to read json, databases size overflow, max database is %d\n",
            MAX_DB_COUNT);
3581 3582 3583 3584 3585 3586 3587 3588
    goto PARSE_OVER;
  }

  g_Dbs.dbCount = dbSize;
  for (int i = 0; i < dbSize; ++i) {
    cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
    if (dbinfos == NULL) continue;

3589
    // dbinfo
3590 3591
    cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
    if (!dbinfo || dbinfo->type != cJSON_Object) {
3592
      printf("ERROR: failed to read json, dbinfo not found\n");
3593 3594
      goto PARSE_OVER;
    }
3595

3596 3597
    cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
    if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
3598
      printf("ERROR: failed to read json, db name not found\n");
3599 3600
      goto PARSE_OVER;
    }
3601
    tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE);
3602 3603 3604

    cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
    if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
3605 3606
      if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) {
        g_Dbs.db[i].drop = true;
3607
      } else {
3608
        g_Dbs.db[i].drop = false;
3609
      }
3610
    } else if (!drop) {
3611
      g_Dbs.db[i].drop = g_args.drop_database;
3612
    } else {
3613
      errorPrint("%s() LN%d, failed to read json, drop input mistake\n",
3614
              __func__, __LINE__);
3615 3616 3617 3618
      goto PARSE_OVER;
    }

    cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
3619
    if (precision && precision->type == cJSON_String
3620
            && precision->valuestring != NULL) {
3621
      tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
3622
              MAX_DB_NAME_SIZE);
3623
    } else if (!precision) {
3624
      //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
3625 3626
      memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE);
    } else {
3627
      printf("ERROR: failed to read json, precision not found\n");
3628 3629 3630 3631 3632 3633 3634 3635 3636
      goto PARSE_OVER;
    }

    cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
    if (update && update->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.update = update->valueint;
    } else if (!update) {
      g_Dbs.db[i].dbCfg.update = -1;
    } else {
3637
      printf("ERROR: failed to read json, update not found\n");
3638 3639 3640 3641 3642 3643 3644 3645 3646
      goto PARSE_OVER;
    }

    cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
    if (replica && replica->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.replica = replica->valueint;
    } else if (!replica) {
      g_Dbs.db[i].dbCfg.replica = -1;
    } else {
3647
      printf("ERROR: failed to read json, replica not found\n");
3648 3649 3650 3651 3652 3653 3654 3655 3656
      goto PARSE_OVER;
    }

    cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
    if (keep && keep->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.keep = keep->valueint;
    } else if (!keep) {
      g_Dbs.db[i].dbCfg.keep = -1;
    } else {
3657 3658
     printf("ERROR: failed to read json, keep not found\n");
     goto PARSE_OVER;
3659
    }
3660

3661 3662 3663 3664 3665 3666
    cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
    if (days && days->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.days = days->valueint;
    } else if (!days) {
      g_Dbs.db[i].dbCfg.days = -1;
    } else {
3667 3668
     printf("ERROR: failed to read json, days not found\n");
     goto PARSE_OVER;
3669
    }
3670

3671 3672 3673 3674 3675 3676
    cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
    if (cache && cache->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cache = cache->valueint;
    } else if (!cache) {
      g_Dbs.db[i].dbCfg.cache = -1;
    } else {
3677 3678
     printf("ERROR: failed to read json, cache not found\n");
     goto PARSE_OVER;
3679
    }
3680

3681 3682 3683 3684 3685 3686
    cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
    if (blocks && blocks->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
    } else if (!blocks) {
      g_Dbs.db[i].dbCfg.blocks = -1;
    } else {
3687 3688
     printf("ERROR: failed to read json, block not found\n");
     goto PARSE_OVER;
3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704
    }

    //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
    //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
    //} else if (!maxtablesPerVnode) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
    //} else {
    // printf("failed to read json, maxtablesPerVnode not found");
    // goto PARSE_OVER;
    //}

    cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
    if (minRows && minRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
    } else if (!minRows) {
3705
      g_Dbs.db[i].dbCfg.minRows = 0;    // 0 means default
3706
    } else {
3707 3708
     printf("ERROR: failed to read json, minRows not found\n");
     goto PARSE_OVER;
3709 3710 3711 3712 3713 3714
    }

    cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
    if (maxRows && maxRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
    } else if (!maxRows) {
3715
      g_Dbs.db[i].dbCfg.maxRows = 0;    // 0 means default
3716
    } else {
3717 3718
     printf("ERROR: failed to read json, maxRows not found\n");
     goto PARSE_OVER;
3719 3720 3721 3722 3723 3724 3725 3726
    }

    cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
    if (comp && comp->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.comp = comp->valueint;
    } else if (!comp) {
      g_Dbs.db[i].dbCfg.comp = -1;
    } else {
3727 3728
     printf("ERROR: failed to read json, comp not found\n");
     goto PARSE_OVER;
3729 3730 3731 3732 3733 3734 3735 3736
    }

    cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
    if (walLevel && walLevel->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
    } else if (!walLevel) {
      g_Dbs.db[i].dbCfg.walLevel = -1;
    } else {
3737 3738
     printf("ERROR: failed to read json, walLevel not found\n");
     goto PARSE_OVER;
3739 3740
    }

3741 3742 3743 3744 3745 3746
    cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
    if (cacheLast && cacheLast->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
    } else if (!cacheLast) {
      g_Dbs.db[i].dbCfg.cacheLast = -1;
    } else {
3747 3748
     printf("ERROR: failed to read json, cacheLast not found\n");
     goto PARSE_OVER;
3749 3750
    }

3751 3752 3753 3754
    cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
    if (quorum && quorum->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
    } else if (!quorum) {
3755
      g_Dbs.db[i].dbCfg.quorum = 1;
3756
    } else {
3757 3758
     printf("failed to read json, quorum input mistake");
     goto PARSE_OVER;
3759 3760 3761 3762 3763 3764 3765 3766
    }

    cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
    if (fsync && fsync->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
    } else if (!fsync) {
      g_Dbs.db[i].dbCfg.fsync = -1;
    } else {
3767
      errorPrint("%s() LN%d, failed to read json, fsync input mistake\n",
3768
              __func__, __LINE__);
3769
      goto PARSE_OVER;
3770
    }
3771

3772
    // super_talbes
3773 3774
    cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
    if (!stables || stables->type != cJSON_Array) {
3775
      errorPrint("%s() LN%d, failed to read json, super_tables not found\n",
3776
              __func__, __LINE__);
3777
      goto PARSE_OVER;
3778 3779
    }

3780 3781
    int stbSize = cJSON_GetArraySize(stables);
    if (stbSize > MAX_SUPER_TABLE_COUNT) {
3782
      errorPrint(
3783 3784
              "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n",
              __func__, __LINE__, MAX_SUPER_TABLE_COUNT);
3785 3786 3787 3788 3789 3790 3791
      goto PARSE_OVER;
    }

    g_Dbs.db[i].superTblCount = stbSize;
    for (int j = 0; j < stbSize; ++j) {
      cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
      if (stbInfo == NULL) continue;
3792 3793

      // dbinfo
3794 3795
      cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
      if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) {
3796
        errorPrint("%s() LN%d, failed to read json, stb name not found\n",
3797
                __func__, __LINE__);
3798 3799
        goto PARSE_OVER;
      }
3800
      tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE);
3801

3802 3803
      cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
      if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
3804
        printf("ERROR: failed to read json, childtable_prefix not found\n");
3805 3806
        goto PARSE_OVER;
      }
3807
      tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE);
3808 3809

      cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null
3810
      if (autoCreateTbl
3811 3812
              && autoCreateTbl->type == cJSON_String
              && autoCreateTbl->valuestring != NULL) {
3813 3814 3815 3816 3817 3818 3819 3820 3821 3822
        if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
        } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        } else {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        }
      } else if (!autoCreateTbl) {
        g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      } else {
3823
        printf("ERROR: failed to read json, auto_create_table not found\n");
3824 3825
        goto PARSE_OVER;
      }
3826

3827 3828 3829 3830
      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
      } else if (!batchCreateTbl) {
3831
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
3832
      } else {
3833
        printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
3834
        goto PARSE_OVER;
3835
      }
3836 3837

      cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
3838
      if (childTblExists
3839 3840
              && childTblExists->type == cJSON_String
              && childTblExists->valuestring != NULL) {
3841 3842
        if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3))
            && (g_Dbs.db[i].drop == false)) {
3843
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
3844
        } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2)
3845
              || (g_Dbs.db[i].drop == true))) {
3846 3847 3848 3849 3850 3851 3852
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        } else {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        }
      } else if (!childTblExists) {
        g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
      } else {
3853
        errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n",
3854
                __func__, __LINE__);
3855 3856
        goto PARSE_OVER;
      }
3857

3858 3859
      cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
      if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
3860
        errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n",
3861
                __func__, __LINE__);
3862 3863 3864 3865 3866
        goto PARSE_OVER;
      }
      g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

      cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
3867
      if (dataSource && dataSource->type == cJSON_String
3868
              && dataSource->valuestring != NULL) {
3869
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
3870
                dataSource->valuestring, MAX_DB_NAME_SIZE);
3871
      } else if (!dataSource) {
3872
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE);
3873
      } else {
3874
        errorPrint("%s() LN%d, failed to read json, data_source not found\n",
3875
            __func__, __LINE__);
3876 3877 3878
        goto PARSE_OVER;
      }

3879
      cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt
3880
      if (insertMode && insertMode->type == cJSON_String
3881
              && insertMode->valuestring != NULL) {
3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892
        if (0 == strcasecmp(insertMode->valuestring, "taosc")) {
            g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE;
        } else if (0 == strcasecmp(insertMode->valuestring, "rest")) {
            g_Dbs.db[i].superTbls[j].insertMode = REST_IFACE;
        } else if (0 == strcasecmp(insertMode->valuestring, "stmt")) {
            g_Dbs.db[i].superTbls[j].insertMode = STMT_IFACE;
        } else {
            errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n",
                    __func__, __LINE__, insertMode->valuestring);
            goto PARSE_OVER;
        }
3893
      } else if (!insertMode) {
3894
        g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE;
3895
      } else {
3896
        errorPrint("%s", "failed to read json, insert_mode not found\n");
3897 3898 3899
        goto PARSE_OVER;
      }

3900
      cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
3901 3902
      if ((childTbl_limit) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3903
        if (childTbl_limit->type != cJSON_Number) {
3904 3905
            printf("ERROR: failed to read json, childtable_limit\n");
            goto PARSE_OVER;
3906 3907 3908
        }
        g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
      } else {
3909
        g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid.
3910 3911 3912
      }

      cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
3913 3914
      if ((childTbl_offset) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3915
        if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) {
3916 3917
            printf("ERROR: failed to read json, childtable_offset\n");
            goto PARSE_OVER;
3918 3919 3920 3921 3922 3923
        }
        g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblOffset = 0;
      }

3924 3925
      cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
      if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
3926
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
3927
                ts->valuestring, MAX_DB_NAME_SIZE);
3928
      } else if (!ts) {
3929
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
3930
                "now", MAX_DB_NAME_SIZE);
3931
      } else {
3932
        printf("ERROR: failed to read json, start_timestamp not found\n");
3933 3934
        goto PARSE_OVER;
      }
3935

3936 3937 3938 3939
      cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
      if (timestampStep && timestampStep->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
      } else if (!timestampStep) {
3940
        g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
3941
      } else {
3942
        printf("ERROR: failed to read json, timestamp_step not found\n");
3943 3944 3945 3946
        goto PARSE_OVER;
      }

      cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
3947
      if (sampleFormat && sampleFormat->type
3948
              == cJSON_String && sampleFormat->valuestring != NULL) {
3949
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
3950
                sampleFormat->valuestring, MAX_DB_NAME_SIZE);
3951
      } else if (!sampleFormat) {
3952
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE);
3953
      } else {
3954
        printf("ERROR: failed to read json, sample_format not found\n");
3955
        goto PARSE_OVER;
3956
      }
3957

3958
      cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
3959 3960
      if (sampleFile && sampleFile->type == cJSON_String
          && sampleFile->valuestring != NULL) {
3961
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
3962
                sampleFile->valuestring, MAX_FILE_NAME_LEN);
3963 3964 3965
      } else if (!sampleFile) {
        memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
      } else {
3966
        printf("ERROR: failed to read json, sample_file not found\n");
3967
        goto PARSE_OVER;
3968
      }
3969

3970 3971
      cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
      if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) {
3972
        tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
3973
                tagsFile->valuestring, MAX_FILE_NAME_LEN);
3974 3975 3976 3977 3978 3979 3980 3981 3982
        if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
          g_Dbs.db[i].superTbls[j].tagSource = 0;
        } else {
          g_Dbs.db[i].superTbls[j].tagSource = 1;
        }
      } else if (!tagsFile) {
        memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
        g_Dbs.db[i].superTbls[j].tagSource = 0;
      } else {
3983
        printf("ERROR: failed to read json, tags_file not found\n");
3984 3985
        goto PARSE_OVER;
      }
3986

sangshuduo's avatar
sangshuduo 已提交
3987 3988 3989
      cJSON* stbMaxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
      if (stbMaxSqlLen && stbMaxSqlLen->type == cJSON_Number) {
        int32_t len = stbMaxSqlLen->valueint;
3990 3991
        if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
          len = TSDB_MAX_ALLOWED_SQL_LEN;
3992 3993
        } else if (len < 5) {
          len = 5;
3994
        }
3995 3996
        g_Dbs.db[i].superTbls[j].maxSqlLen = len;
      } else if (!maxSqlLen) {
3997
        g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len;
3998
      } else {
sangshuduo's avatar
sangshuduo 已提交
3999
        errorPrint("%s() LN%d, failed to read json, stbMaxSqlLen input mistake\n",
4000
            __func__, __LINE__);
4001
        goto PARSE_OVER;
4002
      }
4003
/*
4004 4005 4006 4007 4008
      cJSON *multiThreadWriteOneTbl =
          cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
      if (multiThreadWriteOneTbl
              && multiThreadWriteOneTbl->type == cJSON_String
              && multiThreadWriteOneTbl->valuestring != NULL) {
4009 4010 4011 4012
        if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
        } else {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
4013
        }
4014 4015 4016
      } else if (!multiThreadWriteOneTbl) {
        g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
      } else {
4017
        printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
4018 4019
        goto PARSE_OVER;
      }
4020
*/
sangshuduo's avatar
sangshuduo 已提交
4021 4022 4023
      cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows");
      if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) {
        if (stbInterlaceRows->valueint < 0) {
4024 4025 4026 4027
          errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n",
            __func__, __LINE__);
          goto PARSE_OVER;
        }
sangshuduo's avatar
sangshuduo 已提交
4028
        g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint;
4029 4030
        // rows per table need be less than insert batch
        if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) {
4031
          printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
4032
                  i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR);
4033
          printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
4034
                  g_args.num_of_RPR);
4035
          prompt();
4036 4037
          g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR;
        }
sangshuduo's avatar
sangshuduo 已提交
4038
      } else if (!stbInterlaceRows) {
4039
        g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
4040
      } else {
4041
        errorPrint(
4042 4043
                "%s() LN%d, failed to read json, interlace rows input mistake\n",
                __func__, __LINE__);
4044
        goto PARSE_OVER;
4045
      }
4046 4047 4048

      cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
      if (disorderRatio && disorderRatio->type == cJSON_Number) {
4049 4050 4051 4052 4053 4054
        if (disorderRatio->valueint > 50)
          disorderRatio->valueint = 50;

        if (disorderRatio->valueint < 0)
          disorderRatio->valueint = 0;

4055 4056 4057 4058
        g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
      } else if (!disorderRatio) {
        g_Dbs.db[i].superTbls[j].disorderRatio = 0;
      } else {
4059
        printf("ERROR: failed to read json, disorderRatio not found\n");
4060
        goto PARSE_OVER;
4061
      }
4062 4063 4064 4065 4066 4067 4068

      cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
      if (disorderRange && disorderRange->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
      } else if (!disorderRange) {
        g_Dbs.db[i].superTbls[j].disorderRange = 1000;
      } else {
4069
        printf("ERROR: failed to read json, disorderRange not found\n");
4070 4071
        goto PARSE_OVER;
      }
4072

4073 4074
      cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
      if (insertRows && insertRows->type == cJSON_Number) {
4075 4076 4077 4078 4079
        if (insertRows->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4080 4081 4082 4083
        g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
      } else if (!insertRows) {
        g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
      } else {
4084
        errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
4085
                __func__, __LINE__);
4086 4087 4088 4089 4090 4091
        goto PARSE_OVER;
      }

      cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
      if (insertInterval && insertInterval->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
4092 4093 4094 4095 4096
        if (insertInterval->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4097
      } else if (!insertInterval) {
4098
        verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n",
4099
                __func__, __LINE__, g_args.insert_interval);
4100 4101
        g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
      } else {
4102
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
4103
                __func__, __LINE__);
4104 4105 4106
        goto PARSE_OVER;
      }

4107
      int retVal = getColumnAndTagTypeFromInsertJsonFile(
4108
              stbInfo, &g_Dbs.db[i].superTbls[j]);
4109 4110
      if (false == retVal) {
        goto PARSE_OVER;
4111 4112
      }
    }
4113 4114 4115 4116
  }

  ret = true;

4117
PARSE_OVER:
4118 4119 4120 4121 4122 4123 4124 4125
  return ret;
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
4126
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
4127 4128 4129 4130
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
4131
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE);
4132
  } else if (!host) {
4133
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
4134
  } else {
4135
    printf("ERROR: failed to read json, host not found\n");
4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
4148
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE);
4149
  } else if (!user) {
4150
    tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ;
4151 4152 4153 4154
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
4155
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE);
4156
  } else if (!password) {
4157
    tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);;
4158
  }
H
hzcheng 已提交
4159

4160
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
4161
  if (answerPrompt && answerPrompt->type == cJSON_String
4162
          && answerPrompt->valuestring != NULL) {
4163 4164 4165 4166
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
4167
    } else {
4168
      g_args.answer_yes = false;
H
hzcheng 已提交
4169
    }
4170 4171 4172
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
4173
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
4174
    goto PARSE_OVER;
4175
  }
4176

4177 4178
  cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
  if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
4179
    if (gQueryTimes->valueint <= 0) {
4180 4181
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4182 4183
      goto PARSE_OVER;
    }
4184 4185 4186 4187
    g_args.query_times = gQueryTimes->valueint;
  } else if (!gQueryTimes) {
    g_args.query_times = 1;
  } else {
4188
    errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4189
        __func__, __LINE__);
4190 4191 4192
    goto PARSE_OVER;
  }

4193 4194
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
4195
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE);
4196
  } else if (!dbs) {
4197
    printf("ERROR: failed to read json, databases not found\n");
4198
    goto PARSE_OVER;
H
hzcheng 已提交
4199
  }
4200 4201 4202

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
4203
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
4204
  } else if (!queryMode) {
4205
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
4206
  } else {
4207
    printf("ERROR: failed to read json, query_mode not found\n");
4208
    goto PARSE_OVER;
H
hzcheng 已提交
4209
  }
4210

4211
  // specified_table_query
4212 4213
  cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!specifiedQuery) {
4214
    g_queryInfo.specifiedQueryInfo.concurrent = 1;
4215 4216
    g_queryInfo.specifiedQueryInfo.sqlCount = 0;
  } else if (specifiedQuery->type != cJSON_Object) {
4217
    printf("ERROR: failed to read json, super_table_query not found\n");
4218
    goto PARSE_OVER;
4219
  } else {
4220 4221 4222 4223 4224
    cJSON* queryInterval = cJSON_GetObjectItem(specifiedQuery, "query_interval");
    if (queryInterval && queryInterval->type == cJSON_Number) {
      g_queryInfo.specifiedQueryInfo.queryInterval = queryInterval->valueint;
    } else if (!queryInterval) {
      g_queryInfo.specifiedQueryInfo.queryInterval = 0;
4225 4226
    }

4227
    cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery,
4228
        "query_times");
4229
    if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
4230
      if (specifiedQueryTimes->valueint <= 0) {
4231 4232
        errorPrint(
                "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4233
          __func__, __LINE__, specifiedQueryTimes->valueint);
4234 4235 4236
        goto PARSE_OVER;

      }
4237 4238 4239 4240
      g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint;
    } else if (!specifiedQueryTimes) {
      g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times;
    } else {
4241
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4242
          __func__, __LINE__);
4243
      goto PARSE_OVER;
4244
    }
4245

4246
    cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent");
4247
    if (concurrent && concurrent->type == cJSON_Number) {
4248
      if (concurrent->valueint <= 0) {
4249 4250
        errorPrint(
                "%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n",
4251 4252
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount,
4253
              g_queryInfo.specifiedQueryInfo.concurrent);
4254 4255
        goto PARSE_OVER;
      }
4256
      g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint;
4257
    } else if (!concurrent) {
4258
      g_queryInfo.specifiedQueryInfo.concurrent = 1;
4259
    }
4260

4261 4262 4263 4264 4265 4266 4267
    cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode");
    if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String
        && specifiedAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE;
4268
      } else {
4269
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4270
            __func__, __LINE__);
4271 4272 4273
        goto PARSE_OVER;
      }
    } else {
4274
      g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
4275
    }
4276

4277
    cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval");
4278
    if (interval && interval->type == cJSON_Number) {
4279
      g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint;
4280
    } else if (!interval) {
4281 4282
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4283
      g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000;
4284
    }
4285

4286
    cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart");
4287
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
4288
      if (0 == strcmp("yes", restart->valuestring)) {
4289
        g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4290
      } else if (0 == strcmp("no", restart->valuestring)) {
4291
        g_queryInfo.specifiedQueryInfo.subscribeRestart = false;
4292
      } else {
4293
        printf("ERROR: failed to read json, subscribe restart error\n");
4294 4295 4296
        goto PARSE_OVER;
      }
    } else {
4297
      g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4298
    }
4299

4300
    cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress");
4301
    if (keepProgress
4302 4303
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
4304
      if (0 == strcmp("yes", keepProgress->valuestring)) {
4305
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1;
4306
      } else if (0 == strcmp("no", keepProgress->valuestring)) {
4307
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4308
      } else {
4309
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4310 4311 4312
        goto PARSE_OVER;
      }
    } else {
4313
      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4314 4315
    }

4316
    // sqls
4317
    cJSON* superSqls = cJSON_GetObjectItem(specifiedQuery, "sqls");
4318
    if (!superSqls) {
4319
      g_queryInfo.specifiedQueryInfo.sqlCount = 0;
4320
    } else if (superSqls->type != cJSON_Array) {
4321
      errorPrint("%s() LN%d, failed to read json, super sqls not found\n",
4322
          __func__, __LINE__);
4323
      goto PARSE_OVER;
4324
    } else {
4325 4326
      int superSqlSize = cJSON_GetArraySize(superSqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4327
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
4328
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4329 4330
        goto PARSE_OVER;
      }
4331

4332
      g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize;
4333 4334 4335
      for (int j = 0; j < superSqlSize; ++j) {
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
        if (sql == NULL) continue;
4336

4337 4338
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
4339
          printf("ERROR: failed to read json, sql not found\n");
4340 4341
          goto PARSE_OVER;
        }
4342 4343
        tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j],
                sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
H
hzcheng 已提交
4344

4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356
        cJSON* resubAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume");
        if (resubAfterConsume
                && resubAfterConsume->type == cJSON_Number) {
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j]
                = resubAfterConsume->valueint;
        } else if (!resubAfterConsume) {
            //printf("failed to read json, subscribe interval no found\n");
            //goto PARSE_OVER;
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = 1;
        }

4357
        cJSON *result = cJSON_GetObjectItem(sql, "result");
4358 4359 4360 4361
        if ((NULL != result) && (result->type == cJSON_String)
                && (result->valuestring != NULL)) {
          tstrncpy(g_queryInfo.specifiedQueryInfo.result[j],
                  result->valuestring, MAX_FILE_NAME_LEN);
4362
        } else if (NULL == result) {
4363 4364
          memset(g_queryInfo.specifiedQueryInfo.result[j],
                  0, MAX_FILE_NAME_LEN);
4365
        } else {
4366
          printf("ERROR: failed to read json, super query result file not found\n");
4367
          goto PARSE_OVER;
4368
        }
4369 4370 4371
      }
    }
  }
4372

4373
  // super_table_query
4374 4375
  cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!superQuery) {
4376
    g_queryInfo.superQueryInfo.threadCnt = 1;
4377 4378
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
4379
    printf("ERROR: failed to read json, sub_table_query not found\n");
4380 4381
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
4382
  } else {
4383
    cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval");
4384
    if (subrate && subrate->type == cJSON_Number) {
4385
      g_queryInfo.superQueryInfo.queryInterval = subrate->valueint;
4386
    } else if (!subrate) {
4387
      g_queryInfo.superQueryInfo.queryInterval = 0;
4388 4389 4390 4391
    }

    cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
    if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
4392 4393
      if (superQueryTimes->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4394
          __func__, __LINE__, superQueryTimes->valueint);
4395 4396
        goto PARSE_OVER;
      }
4397 4398 4399 4400
      g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
    } else if (!superQueryTimes) {
      g_queryInfo.superQueryInfo.queryTimes = g_args.query_times;
    } else {
4401
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4402
          __func__, __LINE__);
4403
      goto PARSE_OVER;
4404
    }
4405

4406
    cJSON* threads = cJSON_GetObjectItem(superQuery, "threads");
4407
    if (threads && threads->type == cJSON_Number) {
4408 4409 4410 4411 4412 4413
      if (threads->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, threads input mistake\n",
          __func__, __LINE__);
        goto PARSE_OVER;

      }
4414
      g_queryInfo.superQueryInfo.threadCnt = threads->valueint;
4415
    } else if (!threads) {
4416
      g_queryInfo.superQueryInfo.threadCnt = 1;
4417
    }
4418

4419
    //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count");
4420
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
4421
    //  g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint;
4422
    //} else if (!subTblCnt) {
4423
    //  g_queryInfo.superQueryInfo.childTblCount = 0;
4424
    //}
4425

4426
    cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname");
4427 4428 4429
    if (stblname && stblname->type == cJSON_String
        && stblname->valuestring != NULL) {
      tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring,
4430
          MAX_TB_NAME_SIZE);
4431
    } else {
4432
      errorPrint("%s() LN%d, failed to read json, super table name input error\n",
4433
          __func__, __LINE__);
4434 4435
      goto PARSE_OVER;
    }
4436

4437 4438 4439 4440 4441 4442 4443
    cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode");
    if (superAsyncMode && superAsyncMode->type == cJSON_String
        && superAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE;
4444
      } else {
4445
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4446
            __func__, __LINE__);
4447 4448 4449
        goto PARSE_OVER;
      }
    } else {
4450
      g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
4451
    }
4452

4453 4454 4455 4456 4457 4458 4459 4460 4461
    cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval");
    if (superInterval && superInterval->type == cJSON_Number) {
      if (superInterval->valueint < 0) {
        errorPrint("%s() LN%d, failed to read json, interval input mistake\n",
            __func__, __LINE__);
        goto PARSE_OVER;
      }
      g_queryInfo.superQueryInfo.subscribeInterval = superInterval->valueint;
    } else if (!superInterval) {
4462 4463
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4464
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
4465
    }
4466

4467
    cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart");
4468 4469
    if (subrestart && subrestart->type == cJSON_String
        && subrestart->valuestring != NULL) {
4470
      if (0 == strcmp("yes", subrestart->valuestring)) {
4471
        g_queryInfo.superQueryInfo.subscribeRestart = true;
4472
      } else if (0 == strcmp("no", subrestart->valuestring)) {
4473
        g_queryInfo.superQueryInfo.subscribeRestart = false;
4474
      } else {
4475
        printf("ERROR: failed to read json, subscribe restart error\n");
4476 4477 4478
        goto PARSE_OVER;
      }
    } else {
4479
      g_queryInfo.superQueryInfo.subscribeRestart = true;
4480
    }
4481

4482 4483 4484 4485 4486
    cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
    if (superkeepProgress &&
            superkeepProgress->type == cJSON_String
            && superkeepProgress->valuestring != NULL) {
      if (0 == strcmp("yes", superkeepProgress->valuestring)) {
4487
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
4488
      } else if (0 == strcmp("no", superkeepProgress->valuestring)) {
4489
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4490
      } else {
4491
        printf("ERROR: failed to read json, subscribe super table keepProgress error\n");
4492 4493 4494
        goto PARSE_OVER;
      }
    } else {
4495
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4496
    }
4497

4498
    // sqls
4499
    cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls");
4500
    if (!subsqls) {
4501
      g_queryInfo.superQueryInfo.sqlCount = 0;
4502
    } else if (subsqls->type != cJSON_Array) {
4503
      errorPrint("%s() LN%d: failed to read json, super sqls not found\n",
4504
          __func__, __LINE__);
4505
      goto PARSE_OVER;
4506
    } else {
4507 4508
      int superSqlSize = cJSON_GetArraySize(subsqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4509
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
4510
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4511 4512
        goto PARSE_OVER;
      }
4513

4514
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
4515
      for (int j = 0; j < superSqlSize; ++j) {
4516 4517
        cJSON* sql = cJSON_GetArrayItem(subsqls, j);
        if (sql == NULL) continue;
4518

4519
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
4520 4521 4522
        if (!sqlStr || sqlStr->type != cJSON_String
            || sqlStr->valuestring == NULL) {
          errorPrint("%s() LN%d, failed to read json, sql not found\n",
4523
              __func__, __LINE__);
4524 4525
          goto PARSE_OVER;
        }
4526
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring,
4527
            MAX_QUERY_SQL_LENGTH);
4528

4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540
        cJSON* superResubAfterConsume =
            cJSON_GetObjectItem(sql, "resubAfterConsume");
        if (superResubAfterConsume
                && superResubAfterConsume->type == cJSON_Number) {
            g_queryInfo.superQueryInfo.resubAfterConsume[j] =
                superResubAfterConsume->valueint;
        } else if (!superResubAfterConsume) {
            //printf("failed to read json, subscribe interval no found\n");
            //goto PARSE_OVER;
            g_queryInfo.superQueryInfo.resubAfterConsume[j] = 1;
        }

4541
        cJSON *result = cJSON_GetObjectItem(sql, "result");
4542 4543 4544
        if (result != NULL && result->type == cJSON_String
            && result->valuestring != NULL){
          tstrncpy(g_queryInfo.superQueryInfo.result[j],
4545
              result->valuestring, MAX_FILE_NAME_LEN);
4546
        } else if (NULL == result) {
4547
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4548
        }  else {
4549
          errorPrint("%s() LN%d, failed to read json, sub query result file not found\n",
4550
              __func__, __LINE__);
4551
          goto PARSE_OVER;
4552
        }
4553 4554
      }
    }
H
hzcheng 已提交
4555 4556
  }

4557
  ret = true;
H
hzcheng 已提交
4558

4559
PARSE_OVER:
4560 4561
  return ret;
}
H
hzcheng 已提交
4562

4563
static bool getInfoFromJsonFile(char* file) {
4564
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
4565

4566 4567 4568 4569 4570
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
4571

4572
  bool  ret = false;
4573
  int   maxLen = 6400000;
4574 4575 4576 4577 4578 4579 4580
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
4581
  }
H
Hui Li 已提交
4582

4583 4584 4585
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
4586
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
4587 4588
    goto PARSE_OVER;
  }
H
Hui Li 已提交
4589

4590 4591 4592
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
4593
      g_args.test_mode = INSERT_TEST;
4594
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
4595
      g_args.test_mode = QUERY_TEST;
4596
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
4597
      g_args.test_mode = SUBSCRIBE_TEST;
4598
    } else {
4599
      printf("ERROR: failed to read json, filetype not support\n");
4600 4601 4602
      goto PARSE_OVER;
    }
  } else if (!filetype) {
4603
    g_args.test_mode = INSERT_TEST;
4604
  } else {
4605
    printf("ERROR: failed to read json, filetype not found\n");
4606 4607
    goto PARSE_OVER;
  }
H
hzcheng 已提交
4608

4609
  if (INSERT_TEST == g_args.test_mode) {
4610
    ret = getMetaFromInsertJsonFile(root);
4611
  } else if ((QUERY_TEST == g_args.test_mode)
4612
          || (SUBSCRIBE_TEST == g_args.test_mode)) {
4613 4614
    ret = getMetaFromQueryJsonFile(root);
  } else {
4615
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n",
4616
            __func__, __LINE__);
4617
    goto PARSE_OVER;
4618
  }
4619

4620
PARSE_OVER:
4621 4622
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
4623
  fclose(fp);
4624 4625
  return ret;
}
H
hzcheng 已提交
4626

4627
static void prepareSampleData() {
4628
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4629 4630 4631
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
        (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]);
S
Shuaiqiang Chang 已提交
4632
      }
4633 4634 4635 4636
    }
  }
}

4637
static void postFreeResource() {
4638
  tmfclose(g_fpOfInsertResult);
4639
  for (int i = 0; i < g_Dbs.dbCount; i++) {
S
Shuduo Sang 已提交
4640
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
4641 4642 4643
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
4644
      }
4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4657
    }
4658 4659
  }
}
S
Shuaiqiang Chang 已提交
4660

4661 4662 4663
static int getRowDataFromSample(
        char* dataBuf, int64_t maxLen, int64_t timestamp,
      SSuperTable* superTblInfo, int64_t* sampleUsePos) {
4664
  if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
4665
/*    int ret = readSampleFromCsvFileToMem(superTblInfo);
4666
    if (0 != ret) {
4667
      tmfree(superTblInfo->sampleDataBuf);
4668
      superTblInfo->sampleDataBuf = NULL;
4669 4670
      return -1;
    }
4671
*/
4672 4673
    *sampleUsePos = 0;
  }
S
Shuaiqiang Chang 已提交
4674

4675
  int    dataLen = 0;
4676

4677
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
4678
          "(%" PRId64 ", ", timestamp);
4679
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
4680
          "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
4681
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
S
Shuaiqiang Chang 已提交
4682

4683
  (*sampleUsePos)++;
4684

4685 4686
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4687

4688 4689 4690 4691
static int64_t generateStbRowData(
        SSuperTable* stbInfo,
        char* recBuf, int64_t timestamp
        ) {
4692
  int64_t   dataLen = 0;
4693
  char  *pstr = recBuf;
4694
  int64_t maxLen = MAX_DATA_SIZE;
4695

4696 4697
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
          "(%" PRId64 ",", timestamp);
4698

4699
  for (int i = 0; i < stbInfo->columnCount; i++) {
4700 4701 4702 4703
    if ((0 == strncasecmp(stbInfo->columns[i].dataType,
                    "BINARY", strlen("BINARY")))
            || (0 == strncasecmp(stbInfo->columns[i].dataType,
                    "NCHAR", strlen("NCHAR")))) {
4704
      if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
4705
        errorPrint( "binary or nchar length overflow, max size:%u\n",
4706
                (uint32_t)TSDB_MAX_BINARY_LEN);
4707
        return -1;
4708
      }
4709

4710 4711
      char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
      if (NULL == buf) {
4712
        errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
4713
        return -1;
4714 4715
      }
      rand_string(buf, stbInfo->columns[i].dataLen);
4716
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf);
4717
      tmfree(buf);
4718
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4719
                "INT", 3)) {
4720
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4721
              "%d,", rand_int());
4722
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4723
                "BIGINT", 6)) {
4724
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4725
              "%"PRId64",", rand_bigint());
4726
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4727
                "FLOAT", 5)) {
4728
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4729
              "%f,", rand_float());
4730
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4731
                "DOUBLE", 6)) {
4732
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4733
              "%f,", rand_double());
4734
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4735
                "SMALLINT", 8)) {
4736
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4737
          "%d,", rand_smallint());
4738
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4739
          "TINYINT", strlen("TINYINT"))) {
4740
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4741
          "%d,", rand_tinyint());
4742
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4743
          "BOOL", strlen("BOOL"))) {
4744
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4745
          "%d,", rand_bool());
4746
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4747
          "TIMESTAMP", strlen("TIMESTAMP"))) {
4748
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4749
          "%"PRId64",", rand_bigint());
4750
    }  else {
4751
      errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType);
4752
      return -1;
4753
    }
S
Shuaiqiang Chang 已提交
4754
  }
4755

4756
  dataLen -= 1;
4757
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")");
4758

4759 4760 4761
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

  return strlen(recBuf);
4762
}
S
Shuaiqiang Chang 已提交
4763

4764
static int64_t generateData(char *recBuf, char **data_type,
4765
        int64_t timestamp, int lenOfBinary) {
4766 4767
  memset(recBuf, 0, MAX_DATA_SIZE);
  char *pstr = recBuf;
4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781
  pstr += sprintf(pstr, "(%" PRId64, timestamp);
  int c = 0;

  for (; c < MAX_NUM_DATATYPE; c++) {
    if (data_type[c] == NULL) {
      break;
    }
  }

  if (0 == c) {
    perror("data type error!");
    exit(-1);
  }

H
Haojun Liao 已提交
4782
  for (int i = 0; i < c; i++) {
4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793
    if (strcasecmp(data_type[i % c], "TINYINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_tinyint() );
    } else if (strcasecmp(data_type[i % c], "SMALLINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_smallint());
    } else if (strcasecmp(data_type[i % c], "INT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_int());
    } else if (strcasecmp(data_type[i % c], "BIGINT") == 0) {
      pstr += sprintf(pstr, ",%" PRId64, rand_bigint());
    } else if (strcasecmp(data_type[i % c], "FLOAT") == 0) {
      pstr += sprintf(pstr, ",%10.4f", rand_float());
    } else if (strcasecmp(data_type[i % c], "DOUBLE") == 0) {
4794
      double t = rand_double();
4795 4796
      pstr += sprintf(pstr, ",%20.8f", t);
    } else if (strcasecmp(data_type[i % c], "BOOL") == 0) {
4797
      bool b = rand_bool() & 1;
4798 4799
      pstr += sprintf(pstr, ",%s", b ? "true" : "false");
    } else if (strcasecmp(data_type[i % c], "BINARY") == 0) {
4800 4801
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4802
      pstr += sprintf(pstr, ",\"%s\"", s);
4803
      free(s);
4804
    } else if (strcasecmp(data_type[i % c], "NCHAR") == 0) {
4805 4806
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4807
      pstr += sprintf(pstr, ",\"%s\"", s);
4808 4809 4810
      free(s);
    }

4811
    if (strlen(recBuf) > MAX_DATA_SIZE) {
4812 4813 4814 4815 4816 4817 4818
      perror("column length too long, abort");
      exit(-1);
    }
  }

  pstr += sprintf(pstr, ")");

4819 4820
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

4821
  return (int32_t)strlen(recBuf);
4822 4823
}

4824
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
4825 4826
  char* sampleDataBuf = NULL;

4827
  sampleDataBuf = calloc(
4828
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
4829
  if (sampleDataBuf == NULL) {
4830
      errorPrint("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n",
4831 4832 4833 4834
              __func__, __LINE__,
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE,
              strerror(errno));
      return -1;
4835
  }
4836

4837 4838 4839 4840
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
4841 4842 4843 4844 4845
      errorPrint("%s() LN%d, read sample from csv file failed.\n",
          __func__, __LINE__);
      tmfree(sampleDataBuf);
      superTblInfo->sampleDataBuf = NULL;
      return -1;
H
Hui Li 已提交
4846
  }
4847

4848 4849 4850
  return 0;
}

S
Shuduo Sang 已提交
4851
static int64_t execInsert(threadInfo *pThreadInfo, uint64_t k)
4852 4853
{
  int affectedRows;
4854
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4855

4856
  verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
S
Shuduo Sang 已提交
4857
            __func__, __LINE__, pThreadInfo->buffer);
4858
  if (superTblInfo) {
4859
    if (superTblInfo->insertMode == TAOSC_IFACE) {
S
Shuduo Sang 已提交
4860 4861 4862
      affectedRows = queryDbExec(
              pThreadInfo->taos,
              pThreadInfo->buffer, INSERT_TYPE, false);
4863
    } else if (superTblInfo->insertMode == REST_IFACE) {
4864
      if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port,
S
Shuduo Sang 已提交
4865
                  pThreadInfo->buffer, NULL /* not set result file */)) {
4866
        affectedRows = -1;
4867
        printf("========restful return fail, threadID[%d]\n",
4868
            pThreadInfo->threadID);
4869 4870 4871
      } else {
        affectedRows = k;
      }
4872
    } else if (superTblInfo->insertMode == STMT_IFACE) {
S
Shuduo Sang 已提交
4873 4874 4875 4876 4877 4878
      debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt);
      if (0 != taos_stmt_execute(pThreadInfo->stmt)) {
        errorPrint("%s() LN%d, failied to execute insert statement\n",
                    __func__, __LINE__);
        exit(-1);
      }
S
Shuduo Sang 已提交
4879 4880

      affectedRows = k;
4881
    } else {
4882
      errorPrint("%s() LN%d: unknown insert mode: %d\n",
4883 4884
        __func__, __LINE__, superTblInfo->insertMode);
      affectedRows = 0;
4885
    }
4886
  } else {
S
Shuduo Sang 已提交
4887
    affectedRows = queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, INSERT_TYPE, false);
4888 4889 4890 4891 4892
  }

  return affectedRows;
}

4893 4894
static void getTableName(char *pTblName,
        threadInfo* pThreadInfo, uint64_t tableSeq)
4895 4896
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4897 4898
  if ((superTblInfo)
          && (AUTO_CREATE_SUBTBL != superTblInfo->autoCreateTable)) {
4899
    if (superTblInfo->childTblLimit > 0) {
4900 4901 4902
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName +
            (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
4903
    } else {
4904

4905
        verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n",
4906 4907 4908 4909 4910
                pThreadInfo->threadID, __func__, __LINE__,
                pThreadInfo->start_table_from,
                pThreadInfo->ntables, tableSeq);
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
4911 4912
    }
  } else {
4913
    snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"",
4914
        g_args.tb_prefix, tableSeq);
4915 4916 4917
  }
}

4918 4919 4920 4921 4922
static int64_t generateDataTailWithoutStb(
        uint64_t batch, char* buffer,
        int64_t remainderBufLen, int64_t insertRows,
        uint64_t startFrom, int64_t startTime,
        /* int64_t *pSamplePos, */int64_t *dataLen) {
4923

4924
  uint64_t len = 0;
4925 4926
  char *pstr = buffer;

4927
  verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch);
4928

4929
  int64_t k = 0;
4930
  for (k = 0; k < batch;) {
4931
    char data[MAX_DATA_SIZE];
4932 4933
    memset(data, 0, MAX_DATA_SIZE);

4934
    int64_t retLen = 0;
4935

4936 4937
    char **data_type = g_args.datatype;
    int lenOfBinary = g_args.len_of_binary;
4938

4939 4940 4941 4942 4943 4944 4945 4946 4947
    int64_t randTail = DEFAULT_TIMESTAMP_STEP * k;

    if (g_args.disorderRatio != 0) {
        int rand_num = taosRandom() % 100;
        if (rand_num < g_args.disorderRatio) {
          randTail = (randTail +
                  (taosRandom() % g_args.disorderRange + 1)) * (-1);

          debugPrint("rand data generated, back %"PRId64"\n", randTail);
4948
        }
4949 4950 4951
    } else {
        randTail = DEFAULT_TIMESTAMP_STEP * k;
    }
4952

4953 4954 4955
    retLen = generateData(data, data_type,
            startTime + randTail,
            lenOfBinary);
4956

4957
    if (len > remainderBufLen)
4958
        break;
4959

4960 4961 4962 4963 4964 4965 4966
    pstr += sprintf(pstr, "%s", data);
    k++;
    len += retLen;
    remainderBufLen -= retLen;

    verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n",
            __func__, __LINE__, len, k, buffer);
4967

4968
    startFrom ++;
4969

4970 4971 4972 4973
    if (startFrom >= insertRows) {
      break;
    }
  }
4974

4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015
  *dataLen = len;
  return k;
}

static int64_t generateStbDataTail(
        SSuperTable* superTblInfo,
        uint64_t batch, char* buffer,
        int64_t remainderBufLen, int64_t insertRows,
        uint64_t startFrom, int64_t startTime,
        int64_t *pSamplePos, int64_t *dataLen) {
  uint64_t len = 0;

  char *pstr = buffer;

  verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch);

  int64_t k = 0;
  for (k = 0; k < batch;) {
    char data[MAX_DATA_SIZE];
    memset(data, 0, MAX_DATA_SIZE);

    int64_t retLen = 0;

    if (0 == strncasecmp(superTblInfo->dataSource,
                "sample", strlen("sample"))) {
          retLen = getRowDataFromSample(
                  data,
                  remainderBufLen,
                  startTime + superTblInfo->timeStampStep * k,
                  superTblInfo,
                  pSamplePos);
    } else if (0 == strncasecmp(superTblInfo->dataSource,
                "rand", strlen("rand"))) {
        int64_t randTail = superTblInfo->timeStampStep * k;
        if (superTblInfo->disorderRatio > 0) {
            int rand_num = taosRandom() % 100;
            if(rand_num < superTblInfo->disorderRatio) {
                randTail = (randTail +
                        (taosRandom() % superTblInfo->disorderRange + 1)) * (-1);
                debugPrint("rand data generated, back %"PRId64"\n", randTail);
            }
5016
        }
5017

5018 5019 5020
        int64_t d = startTime + randTail;
        retLen = generateStbRowData(superTblInfo, data, d);
    }
5021

5022
    if (retLen > remainderBufLen) {
5023
        break;
5024 5025
    }

5026 5027 5028 5029 5030
    pstr += snprintf(pstr , retLen + 1, "%s", data);
    k++;
    len += retLen;
    remainderBufLen -= retLen;

5031
    verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n",
5032
            __func__, __LINE__, len, k, buffer);
5033 5034

    startFrom ++;
5035

5036
    if (startFrom >= insertRows) {
5037
      break;
5038
    }
5039
  }
5040

5041 5042 5043
  *dataLen = len;
  return k;
}
5044

5045 5046 5047

static int generateSQLHeadWithoutStb(char *tableName,
        char *dbName,
5048
        char *buffer, int remainderBufLen)
5049 5050
{
  int len;
5051 5052 5053

  char headBuf[HEAD_BUFF_LEN];

5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079
  len = snprintf(
          headBuf,
          HEAD_BUFF_LEN,
          "%s.%s values",
          dbName,
          tableName);

  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

  return len;
}

static int generateStbSQLHead(
        SSuperTable* superTblInfo,
        char *tableName, int32_t tableSeq,
        char *dbName,
        char *buffer, int remainderBufLen)
{
  int len;

  char headBuf[HEAD_BUFF_LEN];

  if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
5080 5081
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
5082
            tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq);
5083
      } else {
5084 5085 5086
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
5087 5088
      }
      if (NULL == tagsValBuf) {
5089
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n",
5090
            __func__, __LINE__);
5091 5092 5093
        return -1;
      }

5094 5095
      len = snprintf(
          headBuf,
5096 5097
                  HEAD_BUFF_LEN,
                  "%s.%s using %s.%s tags %s values",
5098
                  dbName,
5099
                  tableName,
5100
                  dbName,
5101 5102
                  superTblInfo->sTblName,
                  tagsValBuf);
5103 5104
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
5105 5106
      len = snprintf(
          headBuf,
5107 5108
                  HEAD_BUFF_LEN,
                  "%s.%s values",
5109
                  dbName,
5110
                  tableName);
5111
    } else {
5112 5113
      len = snprintf(
          headBuf,
5114 5115
                  HEAD_BUFF_LEN,
                  "%s.%s values",
5116
                  dbName,
5117
                  tableName);
5118 5119
  }

5120 5121 5122 5123 5124
  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

5125 5126 5127
  return len;
}

5128 5129
static int64_t generateStbInterlaceData(
        SSuperTable *superTblInfo,
5130 5131
        char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes,
        uint64_t tableSeq,
5132
        threadInfo *pThreadInfo, char *buffer,
5133
        int64_t insertRows,
5134
        int64_t startTime,
5135
        uint64_t *pRemainderBufLen)
5136
{
5137
  assert(buffer);
5138 5139
  char *pstr = buffer;

5140 5141 5142
  int headLen = generateStbSQLHead(
          superTblInfo,
          tableName, tableSeq, pThreadInfo->db_name,
5143
          pstr, *pRemainderBufLen);
5144 5145 5146 5147 5148

  if (headLen <= 0) {
    return 0;
  }
  // generate data buffer
5149
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n",
5150
            pThreadInfo->threadID, __func__, __LINE__, i, buffer);
5151 5152 5153 5154

  pstr += headLen;
  *pRemainderBufLen -= headLen;

5155
  int64_t dataLen = 0;
5156

5157
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%"PRIu64" batchPerTbl = %"PRIu64"\n",
5158 5159
            pThreadInfo->threadID, __func__, __LINE__,
            i, batchPerTblTimes, batchPerTbl);
5160

5161
  if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
5162
      startTime = taosGetTimestamp(pThreadInfo->time_precision);
5163
  }
5164

5165
  int64_t  k = generateStbDataTail(
5166 5167 5168 5169
            superTblInfo,
            batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
            startTime,
            &(pThreadInfo->samplePos), &dataLen);
5170 5171 5172 5173

  if (k == batchPerTbl) {
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
5174
  } else {
5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209
    debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n",
            __func__, __LINE__, k, batchPerTbl);
    pstr -= headLen;
    pstr[0] = '\0';
    k = 0;
  }

  return k;
}

static int64_t generateInterlaceDataWithoutStb(
        char *tableName, uint64_t batchPerTbl,
        uint64_t tableSeq,
        char *dbName, char *buffer,
        int64_t insertRows,
        uint64_t *pRemainderBufLen)
{
  assert(buffer);
  char *pstr = buffer;

  int headLen = generateSQLHeadWithoutStb(
          tableName, dbName,
            pstr, *pRemainderBufLen);

  if (headLen <= 0) {
    return 0;
  }

  pstr += headLen;
  *pRemainderBufLen -= headLen;

  int64_t dataLen = 0;

  int64_t startTime = 1500000000000;
  int64_t  k = generateDataTailWithoutStb(
5210 5211
            batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
            startTime,
5212
            &dataLen);
5213 5214

  if (k == batchPerTbl) {
5215 5216 5217
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
  } else {
5218
    debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n",
5219
            __func__, __LINE__, k, batchPerTbl);
5220 5221
    pstr -= headLen;
    pstr[0] = '\0';
5222
    k = 0;
5223 5224 5225 5226 5227
  }

  return k;
}

5228 5229
static int64_t generateStbProgressiveData(
        SSuperTable *superTblInfo,
5230
        char *tableName,
5231
        int64_t tableSeq,
5232
        char *dbName, char *buffer,
5233
        int64_t insertRows,
5234
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos,
5235
        int64_t *pRemainderBufLen)
5236 5237
{
  assert(buffer != NULL);
5238
  char *pstr = buffer;
5239

5240
  memset(buffer, 0, *pRemainderBufLen);
5241

5242
  int64_t headLen = generateStbSQLHead(
5243
              superTblInfo,
5244
          tableName, tableSeq, dbName,
5245
          buffer, *pRemainderBufLen);
5246 5247 5248 5249

  if (headLen <= 0) {
    return 0;
  }
5250
  pstr += headLen;
5251
  *pRemainderBufLen -= headLen;
5252

5253
  int64_t dataLen;
5254

5255
  return generateStbDataTail(superTblInfo,
5256 5257
          g_args.num_of_RPR, pstr, *pRemainderBufLen,
          insertRows, startFrom,
5258 5259
          startTime,
          pSamplePos, &dataLen);
5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287
}

static int64_t generateProgressiveDataWithoutStb(
        char *tableName,
        int64_t tableSeq,
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos,
        int64_t *pRemainderBufLen)
{
  assert(buffer != NULL);
  char *pstr = buffer;

  memset(buffer, 0, *pRemainderBufLen);

  int64_t headLen = generateSQLHeadWithoutStb(
          tableName, pThreadInfo->db_name,
          buffer, *pRemainderBufLen);

  if (headLen <= 0) {
    return 0;
  }
  pstr += headLen;
  *pRemainderBufLen -= headLen;

  int64_t dataLen;

  return generateDataTailWithoutStb(
5288 5289 5290
          g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom,
          startTime,
          /*pSamplePos, */&dataLen);
5291
}
5292

5293 5294 5295 5296 5297 5298 5299 5300 5301
static void printStatPerThread(threadInfo *pThreadInfo)
{
  fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n",
          pThreadInfo->threadID,
          pThreadInfo->totalInsertRows,
          pThreadInfo->totalAffectedRows,
          (double)(pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay/1000.0)));
}

5302
// sync write interlace data
5303
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
5304
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
5305 5306
         pThreadInfo->threadID, __func__, __LINE__);

5307
  int64_t insertRows;
5308
  uint64_t interlaceRows;
5309 5310 5311
  uint64_t maxSqlLen;
  int64_t nTimeStampStep;
  uint64_t insert_interval;
5312 5313

  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5314

5315 5316 5317 5318 5319 5320 5321 5322 5323
  if (superTblInfo) {
    insertRows = superTblInfo->insertRows;

    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
5324 5325 5326
    maxSqlLen = superTblInfo->maxSqlLen;
    nTimeStampStep = superTblInfo->timeStampStep;
    insert_interval = superTblInfo->insertInterval;
5327 5328 5329
  } else {
    insertRows = g_args.num_of_DPT;
    interlaceRows = g_args.interlace_rows;
5330 5331 5332
    maxSqlLen = g_args.max_sql_len;
    nTimeStampStep = DEFAULT_TIMESTAMP_STEP;
    insert_interval = g_args.insert_interval;
5333
  }
5334

5335 5336 5337 5338 5339
  debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n",
          pThreadInfo->threadID, __func__, __LINE__,
          pThreadInfo->start_table_from,
          pThreadInfo->ntables, insertRows);

5340
  if (interlaceRows > insertRows)
H
Haojun Liao 已提交
5341
    interlaceRows = insertRows;
5342

5343 5344
  if (interlaceRows > g_args.num_of_RPR)
    interlaceRows = g_args.num_of_RPR;
5345

5346 5347
  uint64_t batchPerTbl = interlaceRows;
  uint64_t batchPerTblTimes;
5348

5349 5350 5351
  if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) {
    batchPerTblTimes =
        g_args.num_of_RPR / interlaceRows;
5352
  } else {
5353
    batchPerTblTimes = 1;
5354 5355
  }

S
Shuduo Sang 已提交
5356 5357
  pThreadInfo->buffer = calloc(maxSqlLen, 1);
  if (NULL == pThreadInfo->buffer) {
5358
    errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5359
              __func__, __LINE__, maxSqlLen, strerror(errno));
5360 5361 5362 5363 5364 5365
    return NULL;
  }

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

5366 5367
  uint64_t st = 0;
  uint64_t et = UINT64_MAX;
5368

5369 5370 5371
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5372

5373
  uint64_t tableSeq = pThreadInfo->start_table_from;
5374 5375
  int64_t startTime = pThreadInfo->start_time;

5376
  uint64_t generatedRecPerTbl = 0;
5377
  bool flagSleep = true;
5378
  uint64_t sleepTimeTotal = 0;
5379

5380
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
5381
    if ((flagSleep) && (insert_interval)) {
5382 5383
        st = taosGetTimestampMs();
        flagSleep = false;
5384 5385
    }
    // generate data
S
Shuduo Sang 已提交
5386
    memset(pThreadInfo->buffer, 0, maxSqlLen);
5387
    uint64_t remainderBufLen = maxSqlLen;
5388

S
Shuduo Sang 已提交
5389
    char *pstr = pThreadInfo->buffer;
5390

5391 5392
    int len = snprintf(pstr,
            strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO);
5393 5394 5395
    pstr += len;
    remainderBufLen -= len;

5396
    uint64_t recOfBatch = 0;
5397

5398
    for (uint64_t i = 0; i < batchPerTblTimes; i ++) {
5399 5400
        char tableName[TSDB_TABLE_NAME_LEN];

5401
      getTableName(tableName, pThreadInfo, tableSeq);
5402 5403
      if (0 == strlen(tableName)) {
        errorPrint("[%d] %s() LN%d, getTableName return null\n",
5404
            pThreadInfo->threadID, __func__, __LINE__);
S
Shuduo Sang 已提交
5405
        free(pThreadInfo->buffer);
5406 5407
        return NULL;
      }
5408

5409
      uint64_t oldRemainderLen = remainderBufLen;
5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428

      int64_t generated;
      if (superTblInfo) {
        generated = generateStbInterlaceData(
                superTblInfo,
                tableName, batchPerTbl, i, batchPerTblTimes,
                tableSeq,
                pThreadInfo, pstr,
                insertRows,
                startTime,
                &remainderBufLen);
      } else {
        generated = generateInterlaceDataWithoutStb(
                tableName, batchPerTbl,
                tableSeq,
                pThreadInfo->db_name, pstr,
                insertRows,
                &remainderBufLen);
      }
5429

5430 5431
      debugPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5432
      if (generated < 0) {
5433
        errorPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
5434
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5435
        goto free_of_interlace;
5436 5437
      } else if (generated == 0) {
        break;
5438 5439
      }

5440
      tableSeq ++;
5441
      recOfBatch += batchPerTbl;
5442
      pstr += (oldRemainderLen - remainderBufLen);
5443
//      startTime += batchPerTbl * superTblInfo->timeStampStep;
5444
      pThreadInfo->totalInsertRows += batchPerTbl;
5445
      verbosePrint("[%d] %s() LN%d batchPerTbl=%"PRId64" recOfBatch=%"PRId64"\n",
5446 5447
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
5448

5449
      if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
5450 5451 5452
            // turn to first table
            tableSeq = pThreadInfo->start_table_from;
            generatedRecPerTbl += batchPerTbl;
5453

5454 5455
            startTime = pThreadInfo->start_time
              + generatedRecPerTbl * nTimeStampStep;
5456

5457 5458 5459
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;
5460

5461 5462 5463
            int remainRows = insertRows - generatedRecPerTbl;
            if ((remainRows > 0) && (batchPerTbl > remainRows))
              batchPerTbl = remainRows;
5464

5465 5466 5467
            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
      }
5468

5469
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n",
5470 5471
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);
5472 5473

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
5474 5475 5476
        break;
    }

5477
    verbosePrint("[%d] %s() LN%d recOfBatch=%"PRIu64" totalInsertRows=%"PRIu64"\n",
5478 5479
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
5480
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
S
Shuduo Sang 已提交
5481
           pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer);
5482

5483
    startTs = taosGetTimestampMs();
5484

5485 5486 5487 5488 5489 5490 5491
    if (recOfBatch == 0) {
      errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n",
              pThreadInfo->threadID, __func__, __LINE__,
              recOfBatch);
      errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n");
      goto free_of_interlace;
    }
S
Shuduo Sang 已提交
5492
    int64_t affectedRows = execInsert(pThreadInfo, recOfBatch);
5493

5494
    endTs = taosGetTimestampMs();
5495 5496
    uint64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n",
5497
            __func__, __LINE__, delay);
5498 5499 5500
    verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5501 5502 5503 5504 5505 5506

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

5507 5508
    if (recOfBatch != affectedRows) {
        errorPrint("[%d] %s() LN%d execInsert insert %"PRIu64", affected rows: %"PRId64"\n%s\n",
5509
                pThreadInfo->threadID, __func__, __LINE__,
S
Shuduo Sang 已提交
5510
                recOfBatch, affectedRows, pThreadInfo->buffer);
5511
        goto free_of_interlace;
5512
    }
5513

5514
    pThreadInfo->totalAffectedRows += affectedRows;
5515

5516 5517
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
5518
      printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n",
5519 5520 5521
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5522 5523
      lastPrintTime = currentPrintTime;
    }
5524

5525
    if ((insert_interval) && flagSleep) {
5526
      et = taosGetTimestampMs();
5527

5528
      if (insert_interval > (et - st) ) {
S
Shuduo Sang 已提交
5529 5530
        uint64_t sleepTime = insert_interval - (et -st);
        performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n",
5531
                    __func__, __LINE__, sleepTime);
5532 5533 5534
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
5535
    }
5536 5537
  }

5538
free_of_interlace:
S
Shuduo Sang 已提交
5539
  tmfree(pThreadInfo->buffer);
5540
  printStatPerThread(pThreadInfo);
5541 5542 5543
  return NULL;
}

5544
// sync insertion progressive data
5545
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
5546
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
5547

5548
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5549
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
5550 5551 5552 5553 5554 5555
  int64_t timeStampStep =
      superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
  int64_t insertRows =
        (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
  verbosePrint("%s() LN%d insertRows=%"PRId64"\n",
            __func__, __LINE__, insertRows);
5556

S
Shuduo Sang 已提交
5557 5558
  pThreadInfo->buffer = calloc(maxSqlLen, 1);
  if (NULL == pThreadInfo->buffer) {
5559
    errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5560 5561
              maxSqlLen,
              strerror(errno));
5562 5563
    return NULL;
  }
5564

5565 5566 5567
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5568

5569
/*  int insert_interval =
5570
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5571 5572
  uint64_t st = 0;
  uint64_t et = 0xffffffff;
5573
  */
5574

5575 5576
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
5577

5578
  pThreadInfo->samplePos = 0;
5579

5580 5581 5582
  for (uint64_t tableSeq = pThreadInfo->start_table_from;
          tableSeq <= pThreadInfo->end_table_to;
          tableSeq ++) {
5583
    int64_t start_time = pThreadInfo->start_time;
5584

5585
    for (uint64_t i = 0; i < insertRows;) {
5586 5587
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
5588
      verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n",
5589 5590
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);
5591

5592
      int64_t remainderBufLen = maxSqlLen;
S
Shuduo Sang 已提交
5593
      char *pstr = pThreadInfo->buffer;
5594

5595 5596
      int len = snprintf(pstr,
              strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO);
5597 5598 5599 5600

      pstr += len;
      remainderBufLen -= len;

5601 5602 5603 5604 5605 5606 5607 5608 5609 5610
      int64_t generated;
      if (superTblInfo) {
        generated = generateStbProgressiveData(
                superTblInfo,
              tableName, tableSeq, pThreadInfo->db_name, pstr, insertRows,
            i, start_time,
            &(pThreadInfo->samplePos),
            &remainderBufLen);
      } else {
        generated = generateProgressiveDataWithoutStb(
5611 5612 5613 5614
              tableName, tableSeq, pThreadInfo, pstr, insertRows,
            i, start_time,
            &(pThreadInfo->samplePos),
            &remainderBufLen);
5615
      }
5616 5617 5618
      if (generated > 0)
        i += generated;
      else
5619
        goto free_of_progressive;
5620

5621
      start_time +=  generated * timeStampStep;
5622
      pThreadInfo->totalInsertRows += generated;
5623

5624
      startTs = taosGetTimestampMs();
5625

S
Shuduo Sang 已提交
5626
      int64_t affectedRows = execInsert(pThreadInfo, generated);
5627

5628
      endTs = taosGetTimestampMs();
5629
      uint64_t delay = endTs - startTs;
5630
      performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n",
5631
              __func__, __LINE__, delay);
5632 5633 5634
      verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5635

5636 5637 5638 5639
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
5640

5641 5642 5643 5644 5645
      if (affectedRows < 0) {
        errorPrint("%s() LN%d, affected rows: %"PRId64"\n",
                __func__, __LINE__, affectedRows);
        goto free_of_progressive;
      }
5646 5647 5648

      pThreadInfo->totalAffectedRows += affectedRows;

5649 5650
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
5651
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
5652 5653 5654
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5655 5656 5657
        lastPrintTime = currentPrintTime;
      }

5658
      if (i >= insertRows)
5659 5660 5661
        break;
    }   // num_of_DPT

5662 5663
    if ((g_args.verbose_print) &&
      (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) &&
5664 5665
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
5666
          verbosePrint("%s() LN%d samplePos=%"PRId64"\n",
5667
                  __func__, __LINE__, pThreadInfo->samplePos);
5668
    }
5669
  } // tableSeq
5670

5671
free_of_progressive:
S
Shuduo Sang 已提交
5672
  tmfree(pThreadInfo->buffer);
5673
  printStatPerThread(pThreadInfo);
5674
  return NULL;
H
Hui Li 已提交
5675 5676
}

5677 5678
static void* syncWrite(void *sarg) {

5679 5680
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5681

5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693
  int interlaceRows;

  if (superTblInfo) {
    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    interlaceRows = g_args.interlace_rows;
  }
5694

5695
  if (interlaceRows > 0) {
5696
    // interlace mode
5697
    return syncWriteInterlace(pThreadInfo);
5698 5699
  } else {
    // progressive mode
5700
    return syncWriteProgressive(pThreadInfo);
5701 5702 5703
  }
}

5704
static void callBack(void *param, TAOS_RES *res, int code) {
5705 5706
  threadInfo* pThreadInfo = (threadInfo*)param;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5707

5708 5709
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5710
  if (insert_interval) {
5711 5712 5713
    pThreadInfo->et = taosGetTimestampMs();
    if ((pThreadInfo->et - pThreadInfo->st) < insert_interval) {
      taosMsleep(insert_interval - (pThreadInfo->et - pThreadInfo->st)); // ms
5714
    }
H
Hui Li 已提交
5715
  }
5716

5717
  char *buffer = calloc(1, pThreadInfo->superTblInfo->maxSqlLen);
5718
  char data[MAX_DATA_SIZE];
5719
  char *pstr = buffer;
5720 5721
  pstr += sprintf(pstr, "insert into %s.%s%"PRId64" values",
          pThreadInfo->db_name, pThreadInfo->tb_prefix,
5722
          pThreadInfo->start_table_from);
5723 5724 5725 5726 5727 5728 5729
//  if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
  if (pThreadInfo->counter >= g_args.num_of_RPR) {
    pThreadInfo->start_table_from++;
    pThreadInfo->counter = 0;
  }
  if (pThreadInfo->start_table_from > pThreadInfo->end_table_to) {
    tsem_post(&pThreadInfo->lock_sem);
5730 5731 5732 5733
    free(buffer);
    taos_free_result(res);
    return;
  }
5734

5735
  for (int i = 0; i < g_args.num_of_RPR; i++) {
5736
    int rand_num = taosRandom() % 100;
5737
    if (0 != pThreadInfo->superTblInfo->disorderRatio
5738
            && rand_num < pThreadInfo->superTblInfo->disorderRatio) {
5739 5740 5741
      int64_t d = pThreadInfo->lastTs
          - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1);
      generateStbRowData(pThreadInfo->superTblInfo, data, d);
5742
    } else {
5743 5744
      generateStbRowData(pThreadInfo->superTblInfo,
              data, pThreadInfo->lastTs += 1000);
H
Hui Li 已提交
5745
    }
5746
    pstr += sprintf(pstr, "%s", data);
5747
    pThreadInfo->counter++;
H
Hui Li 已提交
5748

5749
    if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
5750
      break;
H
Hui Li 已提交
5751 5752
    }
  }
5753

5754
  if (insert_interval) {
5755
    pThreadInfo->st = taosGetTimestampMs();
5756
  }
5757
  taos_query_a(pThreadInfo->taos, buffer, callBack, pThreadInfo);
5758
  free(buffer);
H
Hui Li 已提交
5759

5760
  taos_free_result(res);
H
Hui Li 已提交
5761 5762
}

5763
static void *asyncWrite(void *sarg) {
5764 5765
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5766

5767 5768 5769
  pThreadInfo->st = 0;
  pThreadInfo->et = 0;
  pThreadInfo->lastTs = pThreadInfo->start_time;
5770

5771
  int insert_interval =
5772
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5773
  if (insert_interval) {
5774
    pThreadInfo->st = taosGetTimestampMs();
H
Hui Li 已提交
5775
  }
5776
  taos_query_a(pThreadInfo->taos, "show databases", callBack, pThreadInfo);
5777

5778
  tsem_wait(&(pThreadInfo->lock_sem));
H
Hui Li 已提交
5779 5780 5781 5782

  return NULL;
}

5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808
static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in *serv_addr)
{
  uint16_t rest_port = port + TSDB_PORT_HTTP;
  struct hostent *server = gethostbyname(host);
  if ((server == NULL) || (server->h_addr == NULL)) {
    errorPrint("%s", "ERROR, no such host");
    return -1;
  }

  debugPrint("h_name: %s\nh_addr=%p\nh_addretype: %s\nh_length: %d\n",
            server->h_name,
            server->h_addr,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

  memset(serv_addr, 0, sizeof(struct sockaddr_in));
  serv_addr->sin_family = AF_INET;
  serv_addr->sin_port = htons(rest_port);
#ifdef WINDOWS
  serv_addr->sin_addr.s_addr = inet_addr(host);
#else
  memcpy(&(serv_addr->sin_addr.s_addr), server->h_addr, server->h_length);
#endif
  return 0;
}

5809
static void startMultiThreadInsertData(int threads, char* db_name,
5810
        char* precision,SSuperTable* superTblInfo) {
5811

5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827
  //TAOS* taos;
  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
  //  if (NULL == taos) {
  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
  //    exit(-1);
  //  }
  //}

  int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
  if (0 != precision[0]) {
    if (0 == strncasecmp(precision, "ms", 2)) {
      timePrec = TSDB_TIME_PRECISION_MILLI;
    }  else if (0 == strncasecmp(precision, "us", 2)) {
      timePrec = TSDB_TIME_PRECISION_MICRO;
    }  else {
5828
      errorPrint("Not support precision: %s\n", precision);
5829 5830 5831 5832
      exit(-1);
    }
  }

5833
  int64_t start_time;
5834 5835
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
5836
        start_time = taosGetTimestamp(timePrec);
5837
    } else {
5838
      if (TSDB_CODE_SUCCESS != taosParseTime(
5839 5840 5841 5842 5843
        superTblInfo->startTimestamp,
        &start_time,
        strlen(superTblInfo->startTimestamp),
        timePrec, 0)) {
          ERROR_EXIT("failed to parse time!\n");
5844
      }
5845
    }
5846
  } else {
5847
     start_time = 1500000000000;
5848 5849
  }

5850
  int64_t start = taosGetTimestampMs();
5851

5852
  // read sample data from file first
5853
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5854
              "sample", strlen("sample")))) {
5855
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5856
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
5857
              __func__, __LINE__);
5858 5859 5860 5861
      exit(-1);
    }
  }

5862
  // read sample data from file first
5863
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5864
              "sample", strlen("sample")))) {
5865
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5866
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
5867
              __func__, __LINE__);
5868 5869 5870 5871
      exit(-1);
    }
  }

5872
  TAOS* taos0 = taos_connect(
5873 5874
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
5875
  if (NULL == taos0) {
5876
    errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
5877
                __func__, __LINE__, taos_errstr(NULL));
5878 5879 5880
    exit(-1);
  }

S
Shuduo Sang 已提交
5881
  int64_t ntables = 0;
S
Shuduo Sang 已提交
5882
  uint64_t startFrom;
5883

5884
  if (superTblInfo) {
5885 5886
    int64_t limit;
    uint64_t offset;
5887

H
Haojun Liao 已提交
5888
    if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) &&
5889
            ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) {
H
Haojun Liao 已提交
5890
      printf("WARNING: offset and limit will not be used since the child tables not exists!\n");
5891 5892
    }

5893
    if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) {
5894 5895
      if ((superTblInfo->childTblLimit < 0)
          || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit)
5896
            > (superTblInfo->childTblCount))) {
5897 5898 5899 5900 5901 5902
        superTblInfo->childTblLimit =
            superTblInfo->childTblCount - superTblInfo->childTblOffset;
      }

      offset = superTblInfo->childTblOffset;
      limit = superTblInfo->childTblLimit;
5903
    } else {
5904 5905
      limit = superTblInfo->childTblCount;
      offset = 0;
5906 5907
    }

5908 5909 5910
    ntables = limit;
    startFrom = offset;

5911 5912 5913 5914
    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
        && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
            > superTblInfo->childTblCount)) {
      printf("WARNING: specified offset + limit > child table count!\n");
5915
      prompt();
5916 5917 5918
    }

    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
5919
            && (0 == superTblInfo->childTblLimit)) {
5920
      printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
5921
      prompt();
5922 5923
    }

5924
    superTblInfo->childTblName = (char*)calloc(1,
5925
        limit * TSDB_TABLE_NAME_LEN);
5926
    if (superTblInfo->childTblName == NULL) {
5927
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
5928
      taos_close(taos0);
5929 5930 5931
      exit(-1);
    }

5932
    int64_t childTblCount;
5933
    getChildNameOfSuperTableWithLimitAndOffset(
5934
        taos0,
5935 5936 5937 5938
        db_name, superTblInfo->sTblName,
        &superTblInfo->childTblName, &childTblCount,
        limit,
        offset);
5939 5940 5941
  } else {
    ntables = g_args.num_of_tables;
    startFrom = 0;
5942
  }
5943

5944
  taos_close(taos0);
5945

S
Shuduo Sang 已提交
5946
  int64_t a = ntables / threads;
5947 5948 5949 5950 5951
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

S
Shuduo Sang 已提交
5952
  int64_t b = 0;
5953 5954 5955 5956
  if (threads != 0) {
    b = ntables % threads;
  }

5957
  if ((superTblInfo)
5958 5959 5960 5961
      && (superTblInfo->insertMode == REST_IFACE)) {
      if (convertHostToServAddr(g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) {
        exit(-1);
      }
5962 5963
  }

5964 5965 5966 5967 5968 5969 5970 5971 5972
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  assert(pids != NULL);

  threadInfo *infos = calloc(1, threads * sizeof(threadInfo));
  assert(infos != NULL);

  memset(pids, 0, threads * sizeof(pthread_t));
  memset(infos, 0, threads * sizeof(threadInfo));

H
Hui Li 已提交
5973
  for (int i = 0; i < threads; i++) {
5974
    threadInfo *t_info = infos + i;
H
Hui Li 已提交
5975 5976
    t_info->threadID = i;
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
5977
    t_info->time_precision = timePrec;
5978 5979 5980
    t_info->superTblInfo = superTblInfo;

    t_info->start_time = start_time;
5981
    t_info->minDelay = UINT64_MAX;
5982

5983
    if ((NULL == superTblInfo) ||
5984
            (superTblInfo->insertMode != REST_IFACE)) {
5985
      //t_info->taos = taos;
5986
      t_info->taos = taos_connect(
5987 5988
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
5989
      if (NULL == t_info->taos) {
5990
        errorPrint(
5991 5992
                "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n",
                __func__, __LINE__,
5993
                taos_errstr(NULL));
5994
        free(infos);
5995 5996
        exit(-1);
      }
5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009

      if ((superTblInfo) && (superTblInfo->insertMode == STMT_IFACE)) {
        t_info->stmt = taos_stmt_init(t_info->taos);
        if (NULL == t_info->stmt) {
            errorPrint(
                "%s() LN%d, failed init stmt, reason: %s\n",
                __func__, __LINE__,
                taos_errstr(NULL));
            free(pids);
            free(infos);
            exit(-1);
        }
      }
6010 6011 6012 6013
    } else {
      t_info->taos = NULL;
    }

6014
/*    if ((NULL == superTblInfo)
6015
            || (0 == superTblInfo->multiThreadWriteOneTbl)) {
6016
            */
6017 6018 6019 6020
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
6021
/*    } else {
6022 6023
      t_info->start_table_from = 0;
      t_info->ntables = superTblInfo->childTblCount;
6024 6025
      t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint();
    }
6026
*/
6027
    tsem_init(&(t_info->lock_sem), 0, 0);
6028
    if (ASYNC_MODE == g_Dbs.asyncMode) {
6029
      pthread_create(pids + i, NULL, asyncWrite, t_info);
6030 6031
    } else {
      pthread_create(pids + i, NULL, syncWrite, t_info);
6032
    }
H
Hui Li 已提交
6033
  }
6034

H
Hui Li 已提交
6035 6036 6037 6038
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

6039 6040 6041 6042
  uint64_t totalDelay = 0;
  uint64_t maxDelay = 0;
  uint64_t minDelay = UINT64_MAX;
  uint64_t cntDelay = 1;
6043 6044
  double  avgDelay = 0;

H
Hui Li 已提交
6045
  for (int i = 0; i < threads; i++) {
6046 6047
    threadInfo *t_info = infos + i;

S
TD-1057  
Shengliang Guan 已提交
6048
    tsem_destroy(&(t_info->lock_sem));
6049 6050 6051 6052

    if (t_info->stmt) {
      taos_stmt_close(t_info->stmt);
    }
6053 6054
    taos_close(t_info->taos);

6055
    debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n",
6056 6057 6058
            __func__, __LINE__,
            t_info->threadID, t_info->totalInsertRows,
            t_info->totalAffectedRows);
6059
    if (superTblInfo) {
6060 6061
        superTblInfo->totalAffectedRows += t_info->totalAffectedRows;
        superTblInfo->totalInsertRows += t_info->totalInsertRows;
6062
    } else {
6063 6064
        g_args.totalAffectedRows += t_info->totalAffectedRows;
        g_args.totalInsertRows += t_info->totalInsertRows;
6065
    }
6066 6067 6068 6069

    totalDelay  += t_info->totalDelay;
    cntDelay   += t_info->cntDelay;
    if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay;
6070
    if (t_info->minDelay < minDelay) minDelay = t_info->minDelay;
H
Hui Li 已提交
6071
  }
6072
  cntDelay -= 1;
H
Hui Li 已提交
6073

6074
  if (cntDelay == 0)    cntDelay = 1;
6075 6076
  avgDelay = (double)totalDelay / cntDelay;

6077 6078
  int64_t end = taosGetTimestampMs();
  int64_t t = end - start;
6079 6080

  if (superTblInfo) {
6081
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
6082 6083 6084 6085
          t / 1000.0, superTblInfo->totalInsertRows,
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
6086 6087 6088 6089

    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
6090 6091 6092 6093
          t / 1000.0, superTblInfo->totalInsertRows,
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
6094
    }
6095
  } else {
6096
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
6097 6098 6099 6100
          t / 1000.0, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          (double)g_args.totalInsertRows / (t / 1000.0));
6101 6102 6103
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
6104 6105 6106 6107
          t * 1000.0, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          (double)g_args.totalInsertRows / (t / 1000.0));
6108
    }
6109
  }
6110

6111
  fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
6112
          avgDelay, maxDelay, minDelay);
6113 6114
  if (g_fpOfInsertResult) {
    fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
6115
          avgDelay, maxDelay, minDelay);
6116
  }
6117

6118 6119
  //taos_close(taos);

H
Hui Li 已提交
6120
  free(pids);
6121
  free(infos);
H
Hui Li 已提交
6122 6123
}

6124
static void *readTable(void *sarg) {
6125
#if 1
6126
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
6127 6128
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
6129
  uint64_t sTime = rinfo->start_time;
H
hzcheng 已提交
6130 6131
  char *tb_prefix = rinfo->tb_prefix;
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
6132
  if (NULL == fp) {
6133
    errorPrint( "fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
H
Hui Li 已提交
6134 6135
    return NULL;
  }
6136

6137
  int64_t num_of_DPT;
6138
/*  if (rinfo->superTblInfo) {
6139 6140
    num_of_DPT = rinfo->superTblInfo->insertRows; //  nrecords_per_table;
  } else {
6141
  */
6142
      num_of_DPT = g_args.num_of_DPT;
6143
//  }
6144

S
Shuduo Sang 已提交
6145 6146
  int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
  int64_t totalData = num_of_DPT * num_of_tables;
6147
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6148 6149 6150 6151 6152

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
S
Shuduo Sang 已提交
6153
  printf("%"PRId64" records:\n", totalData);
H
hzcheng 已提交
6154 6155
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

6156
  for (int j = 0; j < n; j++) {
H
hzcheng 已提交
6157
    double totalT = 0;
6158
    uint64_t count = 0;
6159 6160
    for (int64_t i = 0; i < num_of_tables; i++) {
      sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64,
6161
              aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
6162

6163
      double t = taosGetTimestampMs();
S
Shuaiqiang Chang 已提交
6164 6165
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
6166

S
Shuaiqiang Chang 已提交
6167
      if (code != 0) {
6168
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6169
        taos_free_result(pSql);
H
hzcheng 已提交
6170
        taos_close(taos);
6171
        fclose(fp);
6172
        return NULL;
H
hzcheng 已提交
6173 6174
      }

6175
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6176 6177 6178
        count++;
      }

6179
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6180 6181
      totalT += t;

S
Shuaiqiang Chang 已提交
6182
      taos_free_result(pSql);
H
hzcheng 已提交
6183 6184
    }

6185
    fprintf(fp, "|%10s  |   %"PRId64"   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
6186
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
6187
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
6188
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000);
H
hzcheng 已提交
6189 6190 6191
  }
  fprintf(fp, "\n");
  fclose(fp);
6192
#endif
H
hzcheng 已提交
6193 6194 6195
  return NULL;
}

6196
static void *readMetric(void *sarg) {
6197
#if 1
6198
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
6199 6200 6201
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
6202 6203 6204 6205
  if (NULL == fp) {
    printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
    return NULL;
  }
6206

6207
  int64_t num_of_DPT = rinfo->superTblInfo->insertRows;
S
Shuduo Sang 已提交
6208 6209
  int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
  int64_t totalData = num_of_DPT * num_of_tables;
6210
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6211 6212 6213 6214 6215

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
S
Shuduo Sang 已提交
6216 6217
  printf("%"PRId64" records:\n", totalData);
  fprintf(fp, "Querying On %"PRId64" records:\n", totalData);
H
hzcheng 已提交
6218 6219

  for (int j = 0; j < n; j++) {
6220
    char condition[COND_BUF_LEN] = "\0";
B
Bomin Zhang 已提交
6221
    char tempS[64] = "\0";
H
hzcheng 已提交
6222

6223
    int64_t m = 10 < num_of_tables ? 10 : num_of_tables;
H
hzcheng 已提交
6224

6225
    for (int64_t i = 1; i <= m; i++) {
H
hzcheng 已提交
6226
      if (i == 1) {
6227
        sprintf(tempS, "t1 = %"PRId64"", i);
H
hzcheng 已提交
6228
      } else {
6229
        sprintf(tempS, " or t1 = %"PRId64" ", i);
H
hzcheng 已提交
6230
      }
6231
      strncat(condition, tempS, COND_BUF_LEN - 1);
H
hzcheng 已提交
6232

L
liu0x54 已提交
6233
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
6234 6235 6236 6237

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

6238
      double t = taosGetTimestampMs();
H
hzcheng 已提交
6239

S
Shuaiqiang Chang 已提交
6240 6241 6242 6243
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
6244
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6245
        taos_free_result(pSql);
H
hzcheng 已提交
6246
        taos_close(taos);
6247
        fclose(fp);
6248
        return NULL;
H
hzcheng 已提交
6249 6250
      }
      int count = 0;
6251
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6252 6253
        count++;
      }
6254
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6255

6256
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
6257 6258
              num_of_tables * num_of_DPT / (t * 1000.0), t);
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0);
H
hzcheng 已提交
6259

S
Shuaiqiang Chang 已提交
6260
      taos_free_result(pSql);
H
hzcheng 已提交
6261 6262 6263 6264
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
6265
#endif
H
hzcheng 已提交
6266 6267 6268
  return NULL;
}

6269 6270 6271 6272 6273 6274 6275
static void prompt()
{
  if (!g_args.answer_yes) {
    printf("         Press enter key to continue or Ctrl-C to stop\n\n");
    (void)getchar();
  }
}
H
Hui Li 已提交
6276

6277
static int insertTestProcess() {
6278

6279 6280 6281
  setupForAnsiEscape();
  int ret = printfInsertMeta();
  resetAfterAnsiEscape();
6282

6283 6284 6285
  if (ret == -1)
    exit(EXIT_FAILURE);

6286
  debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
6287 6288
  g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
  if (NULL == g_fpOfInsertResult) {
6289
    errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
6290 6291
    return -1;
  }
6292

6293 6294
  if (g_fpOfInsertResult)
    printfInsertMetaToFile(g_fpOfInsertResult);
6295

6296
  prompt();
6297

6298 6299 6300
  init_rand_data();

  // create database and super tables
6301
  if(createDatabasesAndStables() != 0) {
6302 6303
    if (g_fpOfInsertResult)
      fclose(g_fpOfInsertResult);
6304 6305
    return -1;
  }
6306 6307

  // pretreatement
6308
  prepareSampleData();
6309

6310 6311 6312 6313
  double start;
  double end;

  // create child tables
6314
  start = taosGetTimestampMs();
6315
  createChildTables();
6316
  end = taosGetTimestampMs();
6317

6318
  if (g_totalChildTables > 0) {
6319
    fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6320
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6321 6322
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
6323
            "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6324
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6325
    }
6326
  }
6327

6328
  taosMsleep(1000);
6329
  // create sub threads for inserting data
6330
  //start = taosGetTimestampMs();
6331
  for (int i = 0; i < g_Dbs.dbCount; i++) {
6332 6333
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
S
Shuduo Sang 已提交
6334
        for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6335 6336 6337 6338 6339

          SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];

          if (superTblInfo && (superTblInfo->insertRows > 0)) {
            startMultiThreadInsertData(
6340 6341 6342 6343
              g_Dbs.threadCount,
              g_Dbs.db[i].dbName,
              g_Dbs.db[i].dbCfg.precision,
              superTblInfo);
6344
          }
6345
        }
6346
      }
6347
    } else {
6348
        startMultiThreadInsertData(
6349 6350 6351
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
6352
          NULL);
H
Hui Li 已提交
6353
    }
6354
  }
6355
  //end = taosGetTimestampMs();
6356

6357
  //int64_t    totalInsertRows = 0;
6358
  //int64_t    totalAffectedRows = 0;
6359
  //for (int i = 0; i < g_Dbs.dbCount; i++) {
6360
  //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6361
  //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
6362 6363
  //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
  //}
6364
  //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
6365
  postFreeResource();
6366

6367 6368 6369
  return 0;
}

6370 6371
static void *specifiedTableQuery(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6372

6373
  if (pThreadInfo->taos == NULL) {
6374 6375
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
6376 6377 6378 6379
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
6380 6381
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6382
            pThreadInfo->threadID, taos_errstr(NULL));
6383 6384
      return NULL;
    } else {
6385
      pThreadInfo->taos = taos;
6386 6387 6388
    }
  }

6389 6390
  char sqlStr[MAX_DB_NAME_SIZE + 5];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6391 6392
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6393 6394 6395 6396
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
    return NULL;
  }
6397

6398 6399
  uint64_t st = 0;
  uint64_t et = 0;
6400

6401
  uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes;
6402

6403 6404 6405
  uint64_t totalQueried = 0;
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
6406

6407 6408 6409 6410 6411 6412
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
        sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }

6413
  while(queryTimes --) {
6414
    if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) <
6415
            (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) {
6416
      taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms
6417 6418
    }

6419 6420 6421
    st = taosGetTimestampMs();

    selectAndGetResult(pThreadInfo,
6422
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]);
6423

6424 6425 6426 6427
    et = taosGetTimestampMs();
    printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n",
              taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0);

6428 6429
    totalQueried ++;
    g_queryInfo.specifiedQueryInfo.totalQueried ++;
6430

6431 6432
    uint64_t  currentPrintTime = taosGetTimestampMs();
    uint64_t  endTs = taosGetTimestampMs();
6433
    if (currentPrintTime - lastPrintTime > 30*1000) {
6434
      debugPrint("%s() LN%d, endTs=%"PRIu64"ms, startTs=%"PRIu64"ms\n",
6435
          __func__, __LINE__, endTs, startTs);
6436
      printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.6f\n",
6437 6438
                    pThreadInfo->threadID,
                    totalQueried,
6439
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6440
      lastPrintTime = currentPrintTime;
6441
    }
H
Hui Li 已提交
6442
  }
6443 6444
  return NULL;
}
H
Hui Li 已提交
6445

6446
static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) {
6447 6448
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
6449 6450
  sprintf(subTblName, "%s.%s",
          g_queryInfo.dbName,
6451
          g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
6452 6453

  //printf("inSql: %s\n", inSql);
6454

6455 6456
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
6457
    return;
H
Hui Li 已提交
6458
  }
6459

6460
  tstrncpy(outSql, inSql, pos - inSql + 1);
6461
  //printf("1: %s\n", outSql);
6462
  strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1);
6463
  //printf("2: %s\n", outSql);
6464
  strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1);
6465
  //printf("3: %s\n", outSql);
H
Hui Li 已提交
6466 6467
}

6468
static void *superTableQuery(void *sarg) {
6469
  char sqlstr[MAX_QUERY_SQL_LENGTH];
6470
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6471

6472
  if (pThreadInfo->taos == NULL) {
6473 6474 6475 6476 6477 6478 6479 6480
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6481
            pThreadInfo->threadID, taos_errstr(NULL));
6482 6483
      return NULL;
    } else {
6484
      pThreadInfo->taos = taos;
6485 6486 6487
    }
  }

6488 6489
  uint64_t st = 0;
  uint64_t et = (int64_t)g_queryInfo.superQueryInfo.queryInterval;
6490

6491 6492 6493
  uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes;
  uint64_t totalQueried = 0;
  uint64_t  startTs = taosGetTimestampMs();
6494

6495
  uint64_t  lastPrintTime = taosGetTimestampMs();
6496
  while(queryTimes --) {
6497 6498 6499
    if (g_queryInfo.superQueryInfo.queryInterval
            && (et - st) < (int64_t)g_queryInfo.superQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.superQueryInfo.queryInterval - (et - st)); // ms
6500
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to);
6501
    }
H
Hui Li 已提交
6502

6503
    st = taosGetTimestampMs();
6504
    for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) {
6505
      for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
6506
        memset(sqlstr,0,sizeof(sqlstr));
6507
        replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i);
6508
        if (g_queryInfo.superQueryInfo.result[j][0] != 0) {
6509
          sprintf(pThreadInfo->fp, "%s-%d",
6510
                  g_queryInfo.superQueryInfo.result[j],
6511
                  pThreadInfo->threadID);
6512
        }
6513
        selectAndGetResult(pThreadInfo, sqlstr);
6514 6515 6516 6517 6518 6519 6520

        totalQueried++;
        g_queryInfo.superQueryInfo.totalQueried ++;

        int64_t  currentPrintTime = taosGetTimestampMs();
        int64_t  endTs = taosGetTimestampMs();
        if (currentPrintTime - lastPrintTime > 30*1000) {
6521
          printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.3f\n",
6522 6523
                    pThreadInfo->threadID,
                    totalQueried,
6524
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6525
          lastPrintTime = currentPrintTime;
6526
        }
6527
      }
H
Hui Li 已提交
6528
    }
6529
    et = taosGetTimestampMs();
6530
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%"PRIu64" - %"PRIu64"] once queries duration:%.4fs\n\n",
6531
            taosGetSelfPthreadId(),
6532 6533
            pThreadInfo->start_table_from,
            pThreadInfo->end_table_to,
6534
            (double)(et - st)/1000.0);
6535
  }
6536

6537 6538 6539
  return NULL;
}

6540
static int queryTestProcess() {
6541 6542 6543 6544

  setupForAnsiEscape();
  printfQueryMeta();
  resetAfterAnsiEscape();
6545 6546 6547 6548 6549 6550

  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
6551
          g_queryInfo.port);
6552
  if (taos == NULL) {
6553 6554
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6555 6556 6557
    exit(-1);
  }

6558
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6559 6560
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6561 6562 6563
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6564
  }
6565

6566
  prompt();
6567

6568 6569 6570
  if (g_args.debug_print || g_args.verbose_print) {
    printfQuerySystemInfo(taos);
  }
6571

6572 6573 6574 6575 6576 6577
  if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
    if (convertHostToServAddr(
        g_queryInfo.host, g_queryInfo.port, &g_queryInfo.serv_addr) != 0)
      exit(-1);
  }

6578 6579 6580
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
6581
  int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent;
6582
  uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount;
6583

6584
  uint64_t startTs = taosGetTimestampMs();
6585

6586 6587 6588 6589 6590 6591
  if ((nSqlCount > 0) && (nConcurrent > 0)) {

    pids  = malloc(nConcurrent * nSqlCount * sizeof(pthread_t));
    infos = malloc(nConcurrent * nSqlCount * sizeof(threadInfo));

    if ((NULL == pids) || (NULL == infos)) {
6592
      taos_close(taos);
6593
      ERROR_EXIT("memory allocation failed for create threads\n");
6594
    }
6595

6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616
    for (uint64_t i = 0; i < nSqlCount; i++) {
        for (int j = 0; j < nConcurrent; j++) {
            uint64_t seq = i * nConcurrent + j;
            threadInfo *t_info = infos + seq;
            t_info->threadID = seq;
            t_info->querySeq = i;

            if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {

                char sqlStr[MAX_TB_NAME_SIZE*2];
                sprintf(sqlStr, "use %s", g_queryInfo.dbName);
                verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
                if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) {
                    taos_close(taos);
                    free(infos);
                    free(pids);
                    errorPrint( "use database %s failed!\n\n",
                        g_queryInfo.dbName);
                    return -1;
                }
            }
6617

6618
            t_info->taos = NULL;// TODO: workaround to use separate taos connection;
6619

6620 6621
            pthread_create(pids + seq, NULL, specifiedTableQuery,
                t_info);
6622
        }
6623
    }
6624
  } else {
6625
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
6626
  }
6627

6628 6629
  taos_close(taos);

6630 6631 6632
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
6633 6634 6635 6636
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
    pidsOfSub  = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t));
    infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo));
6637 6638

    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
6639 6640
      free(infos);
      free(pids);
6641

6642
      ERROR_EXIT("memory allocation failed for create threads\n");
6643
    }
6644

S
Shuduo Sang 已提交
6645
    int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
6646
    int threads = g_queryInfo.superQueryInfo.threadCnt;
6647

S
Shuduo Sang 已提交
6648
    int64_t a = ntables / threads;
6649 6650 6651 6652
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
6653

S
Shuduo Sang 已提交
6654
    int64_t b = 0;
6655 6656 6657
    if (threads != 0) {
      b = ntables % threads;
    }
6658

6659
    uint64_t startFrom = 0;
6660
    for (int i = 0; i < threads; i++) {
6661 6662
      threadInfo *t_info = infosOfSub + i;
      t_info->threadID = i;
6663

6664 6665 6666 6667
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
6668
      t_info->taos = NULL; // TODO: workaround to use separate taos connection;
6669
      pthread_create(pidsOfSub + i, NULL, superTableQuery, t_info);
6670 6671
    }

6672
    g_queryInfo.superQueryInfo.threadCnt = threads;
6673
  } else {
6674
    g_queryInfo.superQueryInfo.threadCnt = 0;
6675
  }
6676

6677 6678 6679 6680 6681 6682
  if ((nSqlCount > 0) && (nConcurrent > 0)) {
    for (int i = 0; i < nConcurrent; i++) {
      for (int j = 0; j < nSqlCount; j++) {
        pthread_join(pids[i * nSqlCount + j], NULL);
      }
    }
6683
  }
S
Shuaiqiang Chang 已提交
6684

6685
  tmfree((char*)pids);
6686
  tmfree((char*)infos);
6687

6688
  for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
6689
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
6690
  }
H
Hui Li 已提交
6691

6692
  tmfree((char*)pidsOfSub);
6693
  tmfree((char*)infosOfSub);
6694

6695
//  taos_close(taos);// TODO: workaround to use separate taos connection;
6696
  uint64_t endTs = taosGetTimestampMs();
6697

6698
  uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried +
6699 6700
    g_queryInfo.superQueryInfo.totalQueried;

6701
  fprintf(stderr, "==== completed total queries: %"PRIu64", the QPS of all threads: %10.3f====\n",
6702
          totalQueried,
6703
          (double)(totalQueried/((endTs-startTs)/1000.0)));
6704 6705 6706
  return 0;
}

6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721
static void stable_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
  if (res == NULL || taos_errno(res) != 0) {
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
    return;
  }

  if (param)
    appendResultToFile(res, ((threadInfo *)param)->fp);
  // tao_unscribe() will free result.
}

static void specified_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
6722
  if (res == NULL || taos_errno(res) != 0) {
6723 6724
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
6725 6726
    return;
  }
6727

6728
  if (param)
6729
    appendResultToFile(res, ((threadInfo *)param)->fp);
6730
  // tao_unscribe() will free result.
H
hzcheng 已提交
6731 6732
}

6733
static TAOS_SUB* subscribeImpl(
6734 6735 6736 6737
        QUERY_CLASS class,
        threadInfo *pThreadInfo,
        char *sql, char* topic, bool restart, uint64_t interval)
{
6738
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
6739

6740 6741 6742 6743
  if ((SPECIFIED_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) {
    tsub = taos_subscribe(
            pThreadInfo->taos,
6744
            restart,
6745
            topic, sql, specified_sub_callback, (void*)pThreadInfo,
6746
            g_queryInfo.specifiedQueryInfo.subscribeInterval);
6747 6748 6749 6750 6751 6752 6753
  } else if ((STABLE_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) {
    tsub = taos_subscribe(
            pThreadInfo->taos,
            restart,
            topic, sql, stable_sub_callback, (void*)pThreadInfo,
            g_queryInfo.superQueryInfo.subscribeInterval);
6754
  } else {
6755 6756
    tsub = taos_subscribe(
            pThreadInfo->taos,
6757
            restart,
6758
            topic, sql, NULL, NULL, interval);
6759
  }
6760

6761 6762 6763
  if (tsub == NULL) {
    printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
    return NULL;
6764
  }
6765

6766 6767
  return tsub;
}
H
hzcheng 已提交
6768

6769 6770
static void *superSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6771
  char subSqlstr[MAX_QUERY_SQL_LENGTH];
6772
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT] = {0};
6773
  uint64_t tsubSeq;
H
hzcheng 已提交
6774

6775 6776
  if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
      errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n",
6777 6778 6779 6780 6781
              pThreadInfo->ntables,
              MAX_QUERY_SQL_COUNT);
      exit(-1);
  }

6782 6783 6784 6785 6786 6787 6788 6789 6790
  if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
      errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n",
              g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables,
              g_queryInfo.superQueryInfo.sqlCount,
              pThreadInfo->ntables,
              MAX_QUERY_SQL_COUNT);
      exit(-1);
  }

6791
  if (pThreadInfo->taos == NULL) {
6792 6793 6794 6795 6796 6797 6798 6799
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6800
            pThreadInfo->threadID, taos_errstr(NULL));
6801 6802
      return NULL;
    } else {
6803
      pThreadInfo->taos = taos;
6804 6805 6806
    }
  }

6807 6808
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6809 6810
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6811 6812
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
6813 6814
    return NULL;
  }
6815

S
Shengliang Guan 已提交
6816 6817 6818
  char topic[32] = {0};
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6819 6820

      tsubSeq = i - pThreadInfo->start_table_from;
6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839
      verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n",
              __func__, __LINE__,
              pThreadInfo->threadID,
              pThreadInfo->start_table_from,
              pThreadInfo->end_table_to, i);
      sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"",
              i, pThreadInfo->querySeq);
      memset(subSqlstr, 0, sizeof(subSqlstr));
      replaceChildTblName(
              g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq],
              subSqlstr, i);
      if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
        sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
      }

      debugPrint("%s() LN%d, [%d] subSqlstr: %s\n",
              __func__, __LINE__, pThreadInfo->threadID, subSqlstr);
6840
      tsub[tsubSeq] = subscribeImpl(
6841 6842 6843 6844
              STABLE_CLASS,
              pThreadInfo, subSqlstr, topic,
              g_queryInfo.superQueryInfo.subscribeRestart,
              g_queryInfo.superQueryInfo.subscribeInterval);
6845
      if (NULL == tsub[tsubSeq]) {
6846
        taos_close(pThreadInfo->taos);
6847 6848
        return NULL;
      }
S
Shengliang Guan 已提交
6849
  }
6850 6851

  // start loop to consume result
6852 6853 6854 6855
  int consumed[MAX_QUERY_SQL_COUNT];
  for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) {
    consumed[i] = 0;
  }
6856
  TAOS_RES* res = NULL;
6857

6858
  while(1) {
6859 6860
    for (uint64_t i = pThreadInfo->start_table_from;
            i <= pThreadInfo->end_table_to; i++) {
6861 6862 6863 6864
      tsubSeq = i - pThreadInfo->start_table_from;
      if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
          continue;
      }
6865

6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888
      res = taos_consume(tsub[tsubSeq]);
      if (res) {
          if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
              sprintf(pThreadInfo->fp, "%s-%d",
                      g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
              appendResultToFile(res, pThreadInfo->fp);
          }
          if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
              sprintf(pThreadInfo->fp, "%s-%d",
                      g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
              appendResultToFile(res, pThreadInfo->fp);
          }
          consumed[tsubSeq] ++;

          if ((g_queryInfo.superQueryInfo.subscribeKeepProgress)
                  && (consumed[tsubSeq] >=
                      g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
              printf("keepProgress:%d, resub super table query: %"PRIu64"\n",
                      g_queryInfo.superQueryInfo.subscribeKeepProgress,
                      pThreadInfo->querySeq);
              taos_unsubscribe(tsub[tsubSeq],
6889
                    g_queryInfo.superQueryInfo.subscribeKeepProgress);
6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902
              consumed[tsubSeq]= 0;
              tsub[tsubSeq] = subscribeImpl(
                      STABLE_CLASS,
                      pThreadInfo, subSqlstr, topic,
                      g_queryInfo.superQueryInfo.subscribeRestart,
                      g_queryInfo.superQueryInfo.subscribeInterval
                      );
              if (NULL == tsub[tsubSeq]) {
                  taos_close(pThreadInfo->taos);
                  return NULL;
              }
          }
      }
H
hzcheng 已提交
6903 6904
    }
  }
6905
  taos_free_result(res);
6906

S
Shengliang Guan 已提交
6907 6908
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6909 6910
    tsubSeq = i - pThreadInfo->start_table_from;
    taos_unsubscribe(tsub[tsubSeq], 0);
6911
  }
6912

6913
  taos_close(pThreadInfo->taos);
H
hzcheng 已提交
6914 6915 6916
  return NULL;
}

6917 6918
static void *specifiedSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6919
  TAOS_SUB*  tsub = NULL;
6920

6921
  if (pThreadInfo->taos == NULL) {
6922 6923 6924 6925 6926 6927 6928 6929
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6930
            pThreadInfo->threadID, taos_errstr(NULL));
6931 6932
      return NULL;
    } else {
6933
      pThreadInfo->taos = taos;
6934 6935 6936
    }
  }

6937 6938
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6939
  debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
6940 6941
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6942 6943
    return NULL;
  }
6944

S
Shengliang Guan 已提交
6945
  char topic[32] = {0};
6946 6947 6948 6949 6950 6951 6952 6953 6954 6955
  sprintf(topic, "taosdemo-subscribe-%"PRIu64"", pThreadInfo->querySeq);
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
      sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }
  tsub = subscribeImpl(
                      SPECIFIED_CLASS, pThreadInfo,
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
          topic,
6956
          g_queryInfo.specifiedQueryInfo.subscribeRestart,
6957 6958 6959 6960
          g_queryInfo.specifiedQueryInfo.subscribeInterval);
  if (NULL == tsub) {
      taos_close(pThreadInfo->taos);
      return NULL;
S
Shengliang Guan 已提交
6961
  }
6962

6963 6964
  // start loop to consume result
  TAOS_RES* res = NULL;
6965

6966
  int consumed;
6967

6968
  while(1) {
6969
      if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) {
6970 6971
        continue;
      }
6972

6973
      res = taos_consume(tsub);
6974
      if (res) {
6975 6976 6977 6978 6979 6980 6981
          if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
              sprintf(pThreadInfo->fp, "%s-%d",
                      g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
              appendResultToFile(res, pThreadInfo->fp);
          }

6982 6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997 6998 6999
          consumed ++;
          if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress)
                && (consumed >=
                    g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
              printf("keepProgress:%d, resub specified query: %"PRIu64"\n",
                    g_queryInfo.specifiedQueryInfo.subscribeKeepProgress,
                    pThreadInfo->querySeq);
              consumed = 0;
              taos_unsubscribe(tsub,
                      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
              tsub = subscribeImpl(
                      SPECIFIED_CLASS,
                      pThreadInfo,
                      g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
                      topic,
                      g_queryInfo.specifiedQueryInfo.subscribeRestart,
                      g_queryInfo.specifiedQueryInfo.subscribeInterval);
              if (NULL == tsub) {
7000 7001
                taos_close(pThreadInfo->taos);
                return NULL;
7002 7003
              }
          }
7004 7005 7006
      }
  }
  taos_free_result(res);
7007
  taos_unsubscribe(tsub, 0);
7008
  taos_close(pThreadInfo->taos);
7009

H
hzcheng 已提交
7010 7011 7012
  return NULL;
}

7013
static int subscribeTestProcess() {
7014
  setupForAnsiEscape();
7015
  printfQueryMeta();
7016
  resetAfterAnsiEscape();
7017

7018
  prompt();
7019

7020 7021 7022 7023 7024
  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
7025
          g_queryInfo.port);
7026
  if (taos == NULL) {
7027 7028
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
7029 7030 7031
    exit(-1);
  }

7032
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
7033 7034
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
7035 7036 7037
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
7038 7039
  }

7040 7041
  taos_close(taos); // TODO: workaround to use separate taos connection;

7042 7043
  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
7044 7045 7046 7047 7048

  pthread_t  *pidsOfStable  = NULL;
  threadInfo *infosOfStable = NULL;

  //==== create threads for query for specified table
S
Shengliang Guan 已提交
7049
  if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) {
7050
    debugPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n",
7051
              __func__, __LINE__,
S
Shengliang Guan 已提交
7052 7053 7054 7055 7056 7057 7058 7059
              g_queryInfo.specifiedQueryInfo.sqlCount);
  } else {
    if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) {
        errorPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n",
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount);
        exit(-1);
    }
7060

7061 7062 7063 7064 7065 7066 7067 7068
    pids  = malloc(
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(pthread_t));
    infos = malloc(
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(threadInfo));
S
Shengliang Guan 已提交
7069 7070 7071 7072
    if ((NULL == pids) || (NULL == infos)) {
        errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
        exit(-1);
    }
7073

7074 7075 7076 7077 7078 7079 7080 7081 7082
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
            uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
            threadInfo *t_info = infos + seq;
            t_info->threadID = seq;
            t_info->querySeq = i;
            t_info->taos = NULL;  // TODO: workaround to use separate taos connection;
            pthread_create(pids + seq, NULL, specifiedSubscribe, t_info);
        }
S
Shengliang Guan 已提交
7083
    }
H
hzcheng 已提交
7084
  }
7085

7086 7087 7088 7089 7090 7091 7092
  //==== create threads for super table query
  if (g_queryInfo.superQueryInfo.sqlCount <= 0) {
    printf("%s() LN%d, super table query sqlCount %"PRIu64".\n",
              __func__, __LINE__,
              g_queryInfo.superQueryInfo.sqlCount);
  } else {
    if ((g_queryInfo.superQueryInfo.sqlCount > 0)
7093
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
7094 7095 7096
        pidsOfStable  = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
7097
            sizeof(pthread_t));
7098 7099 7100
        infosOfStable = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
7101
            sizeof(threadInfo));
7102 7103
        if ((NULL == pidsOfStable) || (NULL == infosOfStable)) {
            errorPrint("%s() LN%d, malloc failed for create threads\n",
7104
              __func__, __LINE__);
7105 7106 7107
            // taos_close(taos);
            exit(-1);
        }
7108

7109 7110
        int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
        int threads = g_queryInfo.superQueryInfo.threadCnt;
7111

7112 7113 7114 7115 7116
        int64_t a = ntables / threads;
        if (a < 1) {
            threads = ntables;
            a = 1;
        }
7117

7118 7119 7120 7121
        int64_t b = 0;
        if (threads != 0) {
            b = ntables % threads;
        }
7122

7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139
        for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            uint64_t startFrom = 0;
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                threadInfo *t_info = infosOfStable + seq;
                t_info->threadID = seq;
                t_info->querySeq = i;

                t_info->start_table_from = startFrom;
                t_info->ntables = j<b?a+1:a;
                t_info->end_table_to = j<b?startFrom+a:startFrom+a-1;
                startFrom = t_info->end_table_to + 1;
                t_info->taos = NULL; // TODO: workaround to use separate taos connection;
                pthread_create(pidsOfStable + seq,
                        NULL, superSubscribe, t_info);
            }
        }
7140

7141
        g_queryInfo.superQueryInfo.threadCnt = threads;
7142

7143 7144 7145 7146 7147 7148
        for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                pthread_join(pidsOfStable[seq], NULL);
            }
        }
7149
    }
H
hzcheng 已提交
7150
  }
7151

7152 7153 7154 7155 7156
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
    for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
        uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
        pthread_join(pids[seq], NULL);
    }
7157
  }
H
hzcheng 已提交
7158

7159
  tmfree((char*)pids);
7160
  tmfree((char*)infos);
H
hzcheng 已提交
7161

7162 7163
  tmfree((char*)pidsOfStable);
  tmfree((char*)infosOfStable);
7164
//   taos_close(taos);
7165
  return 0;
H
hzcheng 已提交
7166 7167
}

7168
static void initOfInsertMeta() {
7169
  memset(&g_Dbs, 0, sizeof(SDbs));
7170

7171
  // set default values
7172
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7173
  g_Dbs.port = 6030;
7174 7175
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
7176
  g_Dbs.threadCount = 2;
7177 7178

  g_Dbs.use_metric = g_args.use_metric;
L
Liu Tao 已提交
7179 7180
}

7181
static void initOfQueryMeta() {
7182
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
7183

7184
  // set default values
7185
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7186
  g_queryInfo.port = 6030;
7187 7188
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
L
Liu Tao 已提交
7189 7190
}

7191
static void setParaFromArg(){
7192
  if (g_args.host) {
7193
    tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE);
7194
  } else {
7195
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7196
  }
L
Liu Tao 已提交
7197

7198
  if (g_args.user) {
7199
    tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE);
7200
  }
7201 7202

  if (g_args.password) {
7203
    tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE);
7204 7205
  }

7206 7207
  if (g_args.port) {
    g_Dbs.port = g_args.port;
7208
  }
L
Liu Tao 已提交
7209

7210 7211 7212
  g_Dbs.threadCount = g_args.num_of_threads;
  g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;

7213 7214
  g_Dbs.dbCount = 1;
  g_Dbs.db[0].drop = 1;
7215

7216
  tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE);
7217
  g_Dbs.db[0].dbCfg.replica = g_args.replica;
7218
  tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
7219

7220
  tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
7221 7222 7223 7224 7225

  g_Dbs.use_metric = g_args.use_metric;
  g_Dbs.insert_only = g_args.insert_only;

  g_Dbs.do_aggreFunc = true;
L
Liu Tao 已提交
7226

7227 7228
  char dataString[STRING_LEN];
  char **data_type = g_args.datatype;
7229

7230
  memset(dataString, 0, STRING_LEN);
L
Liu Tao 已提交
7231

7232 7233
  if (strcasecmp(data_type[0], "BINARY") == 0
          || strcasecmp(data_type[0], "BOOL") == 0
7234
          || strcasecmp(data_type[0], "NCHAR") == 0 ) {
7235
    g_Dbs.do_aggreFunc = false;
L
Liu Tao 已提交
7236
  }
H
hzcheng 已提交
7237

7238 7239 7240 7241 7242
  if (g_args.use_metric) {
    g_Dbs.db[0].superTblCount = 1;
    tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE);
    g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
    g_Dbs.threadCount = g_args.num_of_threads;
7243
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
7244
    g_Dbs.asyncMode = g_args.async_mode;
7245

7246 7247 7248 7249
    g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
    g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
    g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
7250
    tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix,
7251 7252
            g_args.tb_prefix, MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
S
Shuduo Sang 已提交
7253
    g_Dbs.db[0].superTbls[0].insertMode = g_args.iface;
7254
    tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp,
7255
            "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
7256
    g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
7257

7258
    g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
7259
    g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len;
7260

7261 7262 7263 7264 7265
    g_Dbs.db[0].superTbls[0].columnCount = 0;
    for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
      if (data_type[i] == NULL) {
        break;
      }
7266

7267
      tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
7268
              data_type[i], MAX_TB_NAME_SIZE);
7269
      g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;
7270 7271
      g_Dbs.db[0].superTbls[0].columnCount++;
    }
7272

7273 7274 7275
    if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
      g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
    } else {
7276 7277 7278 7279
      for (int i = g_Dbs.db[0].superTbls[0].columnCount;
              i < g_args.num_of_CPR; i++) {
        tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
                "INT", MAX_TB_NAME_SIZE);
7280
        g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;
7281 7282 7283
        g_Dbs.db[0].superTbls[0].columnCount++;
      }
    }
7284

7285 7286
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType,
            "INT", MAX_TB_NAME_SIZE);
7287
    g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;
7288

7289 7290
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType,
            "BINARY", MAX_TB_NAME_SIZE);
7291 7292
    g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;
    g_Dbs.db[0].superTbls[0].tagCount = 2;
7293
  } else {
7294 7295
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
    g_Dbs.db[0].superTbls[0].tagCount = 0;
H
hzcheng 已提交
7296
  }
7297 7298 7299 7300 7301 7302
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
7303

7304 7305 7306
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
7307 7308 7309
    exit(-1);
  }

7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

7334
static void querySqlFile(TAOS* taos, char* sqlFile)
7335 7336 7337 7338 7339 7340
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
7341

7342 7343 7344 7345 7346 7347
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

7348
  double t = taosGetTimestampMs();
7349

7350
  while((read_len = tgetline(&line, &line_len, fp)) != -1) {
7351 7352 7353 7354 7355 7356 7357 7358 7359 7360 7361 7362
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
7363
    }
7364 7365

    memcpy(cmd + cmd_len, line, read_len);
7366
    verbosePrint("%s() LN%d cmd: %s\n", __func__, __LINE__, cmd);
7367 7368 7369
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) {
        errorPrint("%s() LN%d, queryDbExec %s failed!\n",
               __func__, __LINE__, cmd);
7370 7371 7372 7373 7374
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
7375 7376
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
7377 7378
  }

7379
  t = taosGetTimestampMs() - t;
7380
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
7381

7382 7383 7384 7385
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
7386 7387
}

7388
static void testMetaFile() {
7389
    if (INSERT_TEST == g_args.test_mode) {
7390 7391
      if (g_Dbs.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
7392

7393
      insertTestProcess();
7394

7395
    } else if (QUERY_TEST == g_args.test_mode) {
7396 7397
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7398

7399
      queryTestProcess();
7400

7401
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
7402 7403
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7404

7405
      subscribeTestProcess();
7406

7407 7408 7409
    }  else {
      ;
    }
7410
}
7411

7412
static void queryResult() {
7413
  // query data
7414

7415 7416 7417 7418 7419
  pthread_t read_id;
  threadInfo *rInfo = malloc(sizeof(threadInfo));
  assert(rInfo);
  rInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
  rInfo->start_table_from = 0;
7420

7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439 7440 7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455 7456
  //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
  if (g_args.use_metric) {
    rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
    rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
    rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
    tstrncpy(rInfo->tb_prefix,
          g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE);
  } else {
    rInfo->ntables = g_args.num_of_tables;
    rInfo->end_table_to = g_args.num_of_tables -1;
    tstrncpy(rInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE);
  }

  rInfo->taos = taos_connect(
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
          g_Dbs.port);
  if (rInfo->taos == NULL) {
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
    free(rInfo);
    exit(-1);
  }

  tstrncpy(rInfo->fp, g_Dbs.resultFile, MAX_FILE_NAME_LEN);

  if (!g_Dbs.use_metric) {
    pthread_create(&read_id, NULL, readTable, rInfo);
  } else {
    pthread_create(&read_id, NULL, readMetric, rInfo);
  }
  pthread_join(read_id, NULL);
  taos_close(rInfo->taos);
  free(rInfo);
7457 7458
}

7459 7460
static void testCmdLine() {

7461 7462 7463 7464 7465 7466 7467 7468 7469 7470
  if (strlen(configDir)) {
    wordexp_t full_path;
    if (wordexp(configDir, &full_path, 0) != 0) {
      errorPrint( "Invalid path %s\n", configDir);
      return;
    }
    taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
    wordfree(&full_path);
  }

7471 7472
  g_args.test_mode = INSERT_TEST;
  insertTestProcess();
7473

7474
  if (false == g_Dbs.insert_only)
7475
    queryResult();
7476 7477
}

7478 7479 7480
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

7481
  debugPrint("meta file: %s\n", g_args.metaFile);
7482 7483 7484

  if (g_args.metaFile) {
    initOfInsertMeta();
7485
    initOfQueryMeta();
7486 7487 7488 7489 7490 7491 7492 7493 7494 7495 7496 7497 7498

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
7499 7500 7501 7502
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
7503
          g_Dbs.port);
7504
      querySqlFile(qtaos, g_args.sqlFile);
7505 7506 7507 7508 7509
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
7510 7511 7512

    if (g_dupstr)
        free(g_dupstr);
H
hzcheng 已提交
7513
  }
7514 7515

  return 0;
H
hzcheng 已提交
7516
}
7517